add l10n and added reserved tag list

CLAUDE.md

@@ -87,6 +87,14 @@ arity, or whether a tag is a block tag. Two fallback rules accept them:
   a tag is registered unless it needs the parser itself.
 - the last alternative of `filter`, whose name is `$.identifier` rather than one of the builtin literals.
 
+**A name that only exists inside a tag group has to be reserved instead.** `endif`, `else`, `empty` and
+the rest are tokens only in the state their group opens, so where a tag is named the lexer reads them as an
+identifier and `custom_tag` takes them: `{% endif %}` on its own parsed as a tag from some library. They are
+listed in `NAMES_INSIDE_A_TAG_GROUP` and applied through the `tag_name` reserved context, which wraps
+`custom_tag`'s name **and nothing else** — a reserved context replaces the global one inside whatever it
+wraps, so wrapping the whole rule would un-reserve `as` and reject `{% mytag endif %}`. Adding a tag group
+with a new part or end tag means adding its name there too.
+
 **Builtin names must stay keyword-extractable, or the fallbacks swallow them.** `word: $.identifier` turns
 each builtin's name into its own token, which the lexer prefers wherever it is valid, so a builtin commits
 to its own alternative and its argument rules still apply. This is why a filter's name and its `:` are
@@ -98,7 +106,7 @@ which Django itself enforces while it parses the template ("add requires 2 argum
 ### First-party libraries
 
 Django ships `i18n`, `l10n`, `static`, `cache` and `tz` as libraries rather than builtins. Their tags are
-modelled as ordinary tags (so far: `static`), and the grammar **never requires the `{% load %}`**, because
+modelled as ordinary tags (so far: `static`, `l10n`), and the grammar **never requires the `{% load %}`**, because
 an engine can preload a library through `OPTIONS: {"builtins": [...]}`, which makes `{% static "a" %}` valid
 with no load at all.
 
@@ -113,6 +121,9 @@ no ordering predicate to express it. What such a tool needs to know about Django
   this one.
 - `{% load trans from i18n %}` makes only `trans` available, not the rest of the library.
 
+A library's filters go in their own arity table (`L10N_FILTERS_WITHOUT_ARGUMENT`) rather than into the
+builtin ones, so that the builtin parity check still has an exact list to compare against.
+
 Modelling a library name makes it keyword-extracted, so it no longer reaches `custom_tag`. A project that
 registers its own tag under one of these names and a different signature therefore gets a parse error. That
 is accepted: these names are common enough that shadowing them is the mistake.

grammar.js

@@ -221,6 +221,40 @@ const FILTERS_WITH_OPTIONAL_ARGUMENT = [
   "yesno",
 ];
 
+/**
+ * django.templatetags.l10n. A library's filters need the same arity treatment
+ * as a builtin's, but are kept apart from them so that the builtin parity
+ * check has a list to compare against. queries/libraries.scm records which
+ * library a name came from.
+ */
+const L10N_FILTERS_WITHOUT_ARGUMENT = ["localize", "unlocalize"];
+
+/**
+ * The words that head an end tag or a part of a tag group. Each is a token
+ * only inside the group it belongs to, so anywhere else the lexer reads it as
+ * an identifier and custom_tag accepts it: "{% endif %}" on its own parsed as
+ * a tag some library registered. Reserving them where a tag is named makes
+ * them an error again, while leaving them usable as ordinary words elsewhere
+ * ("{{ endif }}" is a variable, and "{% mytag endif %}" an argument).
+ */
+const NAMES_INSIDE_A_TAG_GROUP = [
+  "elif",
+  "else",
+  "empty",
+  "endautoescape",
+  "endblock",
+  "endcomment",
+  "endfilter",
+  "endfor",
+  "endif",
+  "endifchanged",
+  "endlocalize",
+  "endpartialdef",
+  "endspaceless",
+  "endverbatim",
+  "endwith",
+];
+
 /** Django's string constant, which takes any escape but a line break. */
 const STRING = /"(?:[^"\\]|\\[^\n])*"|'(?:[^'\\]|\\[^\n])*'/;
 
@@ -251,6 +285,7 @@ const django = grammar({
   ],
   reserved: {
     global: ($) => ["not", "if", "in", "is", "as", "for", "from"],
+    tag_name: ($) => NAMES_INSIDE_A_TAG_GROUP,
   },
   rules: {
     template: ($) => repeat1(choice($.template_tag, $.content)),
@@ -353,6 +388,7 @@ const django = grammar({
         $.ifchanged_group,
         $.include,
         $.load,
+        $.localize_group,
         $.lorem,
         $.now,
         $.partial,
@@ -376,7 +412,10 @@ const django = grammar({
      */
     filter: ($) =>
       choice(
-        field("name", choice(...FILTERS_WITHOUT_ARGUMENT)),
+        field(
+          "name",
+          choice(...FILTERS_WITHOUT_ARGUMENT, ...L10N_FILTERS_WITHOUT_ARGUMENT),
+        ),
         seq(
           field("name", choice(...FILTERS_WITH_ARGUMENT)),
           ":",
@@ -421,7 +460,7 @@ const django = grammar({
      * is accepted is what Library.simple_tag and Library.inclusion_tag take,
      * which is how a tag is registered unless it needs the parser itself.
      */
-    custom_tag: ($) => simpleTag($, $.identifier),
+    custom_tag: ($) => simpleTag($, reserved("tag_name", $.identifier)),
     cycle: ($) =>
       block(
         "cycle",
@@ -495,6 +534,7 @@ const django = grammar({
       seq(
         block("ifchanged", repeat(part($.filtered_value))),
         optional($.template),
+        optional(seq(block("else"), optional($.template))),
         block("endifchanged"),
       ),
     include: ($) => block("include", part($.filtered_value)),
@@ -507,6 +547,16 @@ const django = grammar({
           repeat1(part($.library)),
         ),
       ),
+    /**
+     * django.templatetags.l10n. The argument is optional and localize_tag
+     * rejects anything but "on" or "off", including a second word.
+     */
+    localize_group: ($) =>
+      seq(
+        block("localize", optional(part(choice("on", "off")))),
+        optional($.template),
+        block("endlocalize"),
+      ),
     lorem: ($) =>
       block(
         "lorem",

queries/highlights.scm

@@ -77,6 +77,8 @@ tag: [
     ("include")
     ("lorem")
     ("load")
+    ("localize")
+    ("endlocalize")
     ("now")
     ("partial")
     ("partialdef")
@@ -124,6 +126,7 @@ tag: [
         ("linebreaksbr")
         ("linenumbers")
         ("ljust")
+        ("localize")
         ("lower")
         ("make_list")
         ("phone2numeric")
@@ -145,6 +148,7 @@ tag: [
         ("truncatechars_html")
         ("truncatewords")
         ("truncatewords_html")
+        ("unlocalize")
         ("unordered_list")
         ("upper")
         ("urlencode")

queries/libraries.scm

@@ -11,10 +11,26 @@
 ; that extends or includes this one, and a tag written before the load is an
 ; error even though the load is in the same file.
 
-[
+; Each pattern captures the name a library registered, rather than the node it
-  (static)
+; heads, so that the capture is both the name to look up and the range to
-  (get_static_prefix)
+; report. A paired tag names the opening tag only.
-  (get_media_prefix)] @library.static
+
+(static
+  tag: ("static") @library.static)
+
+(get_static_prefix
+  tag: ("get_static_prefix") @library.static)
+
+(get_media_prefix
+  tag: ("get_media_prefix") @library.static)
+
+(localize_group
+  tag: ("localize") @library.l10n)
+
+(filter
+  name: [
+    ("localize")
+    ("unlocalize")] @library.l10n)
 
 ; "{% load static %}" makes every name in the library available; "{% load
 ; static from staticfiles %}" makes only the names listed available.

src/grammar.json

@@ -606,6 +606,10 @@
           "type": "SYMBOL",
           "name": "load"
         },
+        {
+          "type": "SYMBOL",
+          "name": "localize_group"
+        },
         {
           "type": "SYMBOL",
           "name": "lorem"
@@ -784,6 +788,14 @@
               {
                 "type": "STRING",
                 "value": "wordcount"
+              },
+              {
+                "type": "STRING",
+                "value": "localize"
+              },
+              {
+                "type": "STRING",
+                "value": "unlocalize"
               }
             ]
           }
@@ -1590,9 +1602,13 @@
         {
           "type": "FIELD",
           "name": "tag",
+          "content": {
+            "type": "RESERVED",
             "content": {
               "type": "SYMBOL",
               "name": "identifier"
+            },
+            "context_name": "tag_name"
           }
         },
         {
@@ -2935,6 +2951,76 @@
             }
           ]
         },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {
+                      "type": "STRING",
+                      "value": "{%"
+                    },
+                    {
+                      "type": "CHOICE",
+                      "members": [
+                        {
+                          "type": "PATTERN",
+                          "value": "[ \\t\\r\\n]+"
+                        },
+                        {
+                          "type": "BLANK"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "FIELD",
+                      "name": "tag",
+                      "content": {
+                        "type": "STRING",
+                        "value": "else"
+                      }
+                    },
+                    {
+                      "type": "CHOICE",
+                      "members": [
+                        {
+                          "type": "PATTERN",
+                          "value": "[ \\t\\r\\n]+"
+                        },
+                        {
+                          "type": "BLANK"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "STRING",
+                      "value": "%}"
+                    }
+                  ]
+                },
+                {
+                  "type": "CHOICE",
+                  "members": [
+                    {
+                      "type": "SYMBOL",
+                      "name": "template"
+                    },
+                    {
+                      "type": "BLANK"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "SEQ",
           "members": [
@@ -3183,6 +3269,143 @@
         }
       ]
     },
+    "localize_group": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "STRING",
+              "value": "{%"
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            },
+            {
+              "type": "FIELD",
+              "name": "tag",
+              "content": {
+                "type": "STRING",
+                "value": "localize"
+              }
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {
+                      "type": "PATTERN",
+                      "value": "[ \\t\\r\\n]+"
+                    },
+                    {
+                      "type": "CHOICE",
+                      "members": [
+                        {
+                          "type": "STRING",
+                          "value": "on"
+                        },
+                        {
+                          "type": "STRING",
+                          "value": "off"
+                        }
+                      ]
+                    }
+                  ]
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            },
+            {
+              "type": "STRING",
+              "value": "%}"
+            }
+          ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "template"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "STRING",
+              "value": "{%"
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            },
+            {
+              "type": "FIELD",
+              "name": "tag",
+              "content": {
+                "type": "STRING",
+                "value": "endlocalize"
+              }
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            },
+            {
+              "type": "STRING",
+              "value": "%}"
+            }
+          ]
+        }
+      ]
+    },
     "lorem": {
       "type": "SEQ",
       "members": [
@@ -4822,6 +5045,68 @@
         "type": "STRING",
         "value": "from"
       }
+    ],
+    "tag_name": [
+      {
+        "type": "STRING",
+        "value": "elif"
+      },
+      {
+        "type": "STRING",
+        "value": "else"
+      },
+      {
+        "type": "STRING",
+        "value": "empty"
+      },
+      {
+        "type": "STRING",
+        "value": "endautoescape"
+      },
+      {
+        "type": "STRING",
+        "value": "endblock"
+      },
+      {
+        "type": "STRING",
+        "value": "endcomment"
+      },
+      {
+        "type": "STRING",
+        "value": "endfilter"
+      },
+      {
+        "type": "STRING",
+        "value": "endfor"
+      },
+      {
+        "type": "STRING",
+        "value": "endif"
+      },
+      {
+        "type": "STRING",
+        "value": "endifchanged"
+      },
+      {
+        "type": "STRING",
+        "value": "endlocalize"
+      },
+      {
+        "type": "STRING",
+        "value": "endpartialdef"
+      },
+      {
+        "type": "STRING",
+        "value": "endspaceless"
+      },
+      {
+        "type": "STRING",
+        "value": "endverbatim"
+      },
+      {
+        "type": "STRING",
+        "value": "endwith"
+      }
     ]
   }
 }

src/node-types.json

@@ -75,6 +75,10 @@
         "type": "load",
         "named": true
       },
+      {
+        "type": "localize_group",
+        "named": true
+      },
       {
         "type": "lorem",
         "named": true
@@ -568,6 +572,10 @@
             "type": "ljust",
             "named": false
           },
+          {
+            "type": "localize",
+            "named": false
+          },
           {
             "type": "lower",
             "named": false
@@ -652,6 +660,10 @@
             "type": "truncatewords_html",
             "named": false
           },
+          {
+            "type": "unlocalize",
+            "named": false
+          },
           {
             "type": "unordered_list",
             "named": false
@@ -968,6 +980,10 @@
         "multiple": true,
         "required": true,
         "types": [
+          {
+            "type": "else",
+            "named": false
+          },
           {
             "type": "endifchanged",
             "named": false
@@ -1088,6 +1104,36 @@
       ]
     }
   },
+  {
+    "type": "localize_group",
+    "named": true,
+    "fields": {
+      "tag": {
+        "multiple": true,
+        "required": true,
+        "types": [
+          {
+            "type": "endlocalize",
+            "named": false
+          },
+          {
+            "type": "localize",
+            "named": false
+          }
+        ]
+      }
+    },
+    "children": {
+      "multiple": false,
+      "required": false,
+      "types": [
+        {
+          "type": "template",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "lorem",
     "named": true,
@@ -1888,6 +1934,10 @@
     "type": "endifchanged",
     "named": false
   },
+  {
+    "type": "endlocalize",
+    "named": false
+  },
   {
     "type": "endpartialdef",
     "named": false
@@ -2032,6 +2082,10 @@
     "type": "load",
     "named": false
   },
+  {
+    "type": "localize",
+    "named": false
+  },
   {
     "type": "lorem",
     "named": false
@@ -2232,6 +2286,10 @@
     "type": "truncatewords_html",
     "named": false
   },
+  {
+    "type": "unlocalize",
+    "named": false
+  },
   {
     "type": "unordered_list",
     "named": false

test/corpus/custom.txt

@@ -229,3 +229,58 @@ Builtin Filter With An Argument It Does Not Take
 {{ x|length:2 }}
 
 ---
+
+==================
+Custom Tag Does Not Cover An End Tag
+:error
+==================
+
+{% endif %}
+
+---
+
+
+==================
+Custom Tag Does Not Cover A Part Of A Tag Group
+:error
+==================
+
+{% else %}
+
+---
+
+
+==================
+A Name Reserved For A Tag Is Still A Variable
+==================
+
+{{ endif }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))
+
+==================
+A Name Reserved For A Tag Is Still An Argument
+==================
+
+{% mytag endif %}
+
+---
+
+(template
+  (content)
+  (custom_tag
+    tag: (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))

test/corpus/ifchanged.txt

@@ -39,3 +39,132 @@ content
     (template
       (content)))
   (content))
+
+==================
+Ifchanged With Else
+==================
+
+{% ifchanged %}
+content
+{% else %}
+unchanged
+{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (template
+      (content))
+    (template
+      (content)))
+  (content))
+
+==================
+Ifchanged With Values And Else
+==================
+
+{% ifchanged x y|upper %}
+content
+{% else %}
+unchanged
+{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter)))
+    (template
+      (content))
+    (template
+      (content)))
+  (content))
+
+==================
+Ifchanged With An Empty Else
+==================
+
+{% ifchanged %}
+content
+{% else %}{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (template
+      (content)))
+  (content))
+
+==================
+Ifchanged Nested In The Else Of Another
+==================
+
+{% ifchanged %}
+a
+{% else %}
+{% ifchanged %}
+b
+{% else %}
+c
+{% endifchanged %}
+{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (template
+      (content))
+    (template
+      (content)
+      (ifchanged_group
+        (template
+          (content))
+        (template
+          (content)))
+      (content)))
+  (content))
+
+==================
+Ifchanged With Two Elses
+:error
+==================
+
+{% ifchanged %}
+a
+{% else %}
+b
+{% else %}
+c
+{% endifchanged %}
+
+---
+
+
+==================
+Ifchanged With An Argument On Else
+:error
+==================
+
+{% ifchanged %}
+a
+{% else x %}
+b
+{% endifchanged %}
+
+---

test/corpus/l10n.txt

@@ -0,0 +1,163 @@
+==================
+Localize
+==================
+
+{% localize %}x{% endlocalize %}
+
+---
+
+(template
+  (content)
+  (localize_group
+    (template
+      (content)))
+  (content))
+
+==================
+Localize On
+==================
+
+{% localize on %}x{% endlocalize %}
+
+---
+
+(template
+  (content)
+  (localize_group
+    (template
+      (content)))
+  (content))
+
+==================
+Localize Off
+==================
+
+{% localize off %}x{% endlocalize %}
+
+---
+
+(template
+  (content)
+  (localize_group
+    (template
+      (content)))
+  (content))
+
+==================
+Localize Hugging Its Delimiters
+==================
+
+{%localize%}x{%endlocalize%}
+
+---
+
+(template
+  (content)
+  (localize_group
+    (template
+      (content)))
+  (content))
+
+==================
+Localize Nested
+==================
+
+{% localize on %}{% localize off %}y{% endlocalize %}{% endlocalize %}
+
+---
+
+(template
+  (content)
+  (localize_group
+    (template
+      (localize_group
+        (template
+          (content)))))
+  (content))
+
+==================
+Localize With Another Argument
+:error
+==================
+
+{% localize maybe %}x{% endlocalize %}
+
+---
+
+
+==================
+Localize With Two Arguments
+:error
+==================
+
+{% localize on off %}x{% endlocalize %}
+
+---
+
+
+==================
+Localize Unclosed
+:error
+==================
+
+{% localize %}
+
+---
+
+
+==================
+Localize Filter
+==================
+
+{{ x|localize }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter))))
+  (content))
+
+==================
+Unlocalize Filter
+==================
+
+{{ x|unlocalize }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter))))
+  (content))
+
+==================
+Localize Filter With An Argument
+:error
+==================
+
+{{ x|localize:1 }}
+
+---
+
+
+==================
+Unlocalize Filter With An Argument
+:error
+==================
+
+{{ x|unlocalize:1 }}
+
+---