add unexpected argument

CLAUDE.md

@@ -204,19 +204,35 @@ from displacing a tag the group does take, and `queries/errors.scm` reports it a
 **Words a tag does not take are an `unexpected_argument` inside it.** Every close utility accepts one
 before its scanner token, so `{% else x %}` stays the `else` of its group, `{% endif junk %}` still closes
 its `if` (it used to leave the whole file an `ERROR`), and `{% static "a.css" "b.css" %}` is a `static_tag`.
-Each word is `token(prec(-1, /[^\s%]+/))`: tree-sitter weighs lexical precedence before match length, so a
+The node is a run of hidden `_unexpected_word`s, and three details make it work:
-token an argument really takes wins wherever one is valid (`b|upper` stays a filtered value) and a loose
-word only matches where nothing else can. The node costs -1 of dynamic precedence, which is why
-`unexpected_tag` costs -2 or -4: a misused tag its group does take has to beat reading it as a stray.
-A separator after any argument can now also start stray words, so it needs `conflicts` entries for
-`unexpected_argument`, `predicate` and `_translate_option`, and it took the table from 3,363 states to
-4,894 and the Wasm build from 251KB to 319KB.
 
-Two consequences. A repeat the scanner refuses — `{% with a=1 a=2 %}`, `{% blocktrans trimmed trimmed %}`
+- **Lexical precedence -1.** A loose word is `token(prec(-1, /[^\s%]+/))`. Tree-sitter weighs lexical
-— is reported as an unexpected argument rather than an `ERROR`. And where a keyword is valid after the
+  precedence before match length, so a token an argument really takes wins wherever one is valid
-arguments (`reversed`, `as`, `only`), a stray word shaped like an identifier is still an `ERROR`:
+  (`b|upper` stays a filtered value) and a loose word only matches where nothing else can.
-`{% for a in b zqx %}` puts it on `b`, most likely because keyword extraction lexes `zqx` as an
+- **Identifiers are words too.** Wherever a keyword is valid — after `{% for a in b`, where `reversed` may
-`identifier`, which is not valid there, before the loose-word token is considered.
+  follow, or after `as`, `only`, `noop`, `silent` — tree-sitter lexes every word as an `identifier` to
+  check for the keyword, and the loose-word token never gets a turn. So a word may also be an
+  `identifier`, aliased to an anonymous `unexpected_word` so `queries/locals.scm` does not count it as a
+  reference, with whatever touches it (`a=2`, `x.y|upper`, `"z"|lower:"q"`) as an immediate tail.
+- **-1 of dynamic precedence per word**, not per node, so a reading that flags more words than it must
+  always loses: `{% cycle a b as x silent zqx %}` flags `zqx`, not everything after `a`. `unexpected_tag`
+  costs -2 or -4 so that a misused tag its group does take beats reading it as a stray.
+
+A tag with a known keyword signature (`simpleTag` given names, so far only `csp_nonce_attr`) needs one more
+piece: the scanner's `_kwarg_name` accepts any name followed by `=` and commits the parser to a keyword
+argument before the name is checked, so `{% csp_nonce_attr other="b" %}` would be an `ERROR`. `simpleTag`
+offers `_unexpected_keyword` — an identifier and its tail, aliased to `unexpected_argument` — beside the
+names it knows.
+
+A separator after any argument can now also start stray words, so this needs `conflicts` entries for
+`unexpected_argument`, `predicate`, `_translate_option`, and `_unexpected_word` with `variable_attribute`,
+`resetcycle_tag` and `library`. It took the table from 3,363 states to 4,913 and the Wasm build from 251KB to
+323KB. Two consequences: a repeat the scanner refuses — `{% with a=1 a=2 %}`, `{% blocktrans trimmed trimmed %}`
+— is reported as an unexpected argument rather than an `ERROR`; and a stray **reserved** word is still an
+`ERROR`, since `not`, `if`, `in`, `is`, `as`, `for` and `from` can never be an identifier:
+`{% trans "hi" as x as y %}` errors on the second `as`. A tag missing a required argument, such as
+`{% with a = 1 %}`, is an `ERROR` too; `unexpected_argument` only takes words after a tag that is otherwise
+complete.
 
 Where several tags share a name the group holds only once — `{% if %}{% else %}{% else %}{% else %}` — reading
 any of them as the stray parses, so the ties are broken by dynamic precedence: `unexpected_tag` is -2 when

grammar.js

@@ -281,7 +281,14 @@ function simpleTag(name, args = true, kwargs = true) {
     const named = Object.entries(kwargs).map(([name, value]) =>
       part(seq(sym("_kwarg_name"), name, "=", value)),
     );
-    parts.push(repeat(choice(...named)));
+    // a name the signature does not take is flagged in place
+    const unexpected = part(
+      seq(
+        sym("_kwarg_name"),
+        alias(sym("_unexpected_keyword"), sym("unexpected_argument")),
+      ),
+    );
+    parts.push(repeat(choice(...named, unexpected)));
   }
   return tag(name, ...parts, optional(asVariable));
 }
@@ -517,6 +524,9 @@ const django = grammar({
     [$.predicate],
     [$._translate_option],
     [$.unexpected_argument],
+    [$.variable_attribute, $._unexpected_word],
+    [$.resetcycle_tag, $._unexpected_word],
+    [$.library, $._unexpected_word],
     [$.library, $.load_tag],
     [$._filtered_value_spaced],
     [$._filter_expression_spaced],
@@ -1218,14 +1228,50 @@ const django = grammar({
      * Words a tag does not take, kept inside the tag they were written in. Every
      * close utility accepts them before its "%}", so a misused tag keeps its own
      * node, "{% else x %}" is still the else of its group, and error recovery
-     * has nothing to repair. Each word is a token of lexical precedence -1, so
+     * has nothing to repair. queries/errors.scm reports it.
-     * that a token an argument really takes wins wherever one is valid, even
-     * over a longer match: "b|upper" is still a filtered value. The node costs
-     * -1 of dynamic precedence, less than any unexpected_tag, and
-     * queries/errors.scm reports it.
      */
-    unexpected_argument: ($) =>
+    unexpected_argument: ($) => repeat1($._unexpected_word),
-      prec.dynamic(-1, repeat1(seq(SEP, token(prec(-1, /[^\s%]+/))))),
+    /**
+     * One word of an unexpected_argument, costing -1 of dynamic precedence of
+     * its own, so that a reading flagging more words than it must always loses:
+     * "{% cycle a b as x silent zqx %}" flags zqx, not everything after a.
+     *
+     * The word is a token of lexical precedence -1, so that a token an argument
+     * really takes wins wherever one is valid, even over a longer match:
+     * "b|upper" is still a filtered value. Where a keyword is valid, as after
+     * "{% for a in b" where "reversed" may follow, tree-sitter lexes every word
+     * as an identifier to check for the keyword, so an identifier is a word
+     * here too, followed by whatever touches it ("a=2", "x.y|upper"). It is
+     * aliased so that queries/locals.scm does not take it for a reference.
+     */
+    _unexpected_word: ($) =>
+      prec.dynamic(
+        -1,
+        seq(
+          SEP,
+          choice(
+            token(prec(-1, /[^\s%]+/)),
+            seq(
+              alias($.identifier, "unexpected_word"),
+              optional(token.immediate(prec(-1, /[^\s%]+/))),
+            ),
+          ),
+        ),
+      ),
+    /**
+     * A "name=value" argument whose name a tag with a known signature does not
+     * take. The scanner's _kwarg_name reads any name followed by "=", which
+     * commits the parser to a keyword argument before the name is known, so
+     * the name the signature lacks is taken here rather than after the tag.
+     */
+    _unexpected_keyword: ($) =>
+      prec.dynamic(
+        -1,
+        seq(
+          alias($.identifier, "unexpected_word"),
+          token.immediate(prec(-1, /[^\s%]+/)),
+        ),
+      ),
     url_tag: ($) =>
       tag(
         "url",

src/grammar.json

@@ -2893,6 +2893,33 @@
                     ]
                   }
                 ]
+              },
+              {
+                "type": "SEQ",
+                "members": [
+                  {
+                    "type": "SYMBOL",
+                    "name": "_sep"
+                  },
+                  {
+                    "type": "SEQ",
+                    "members": [
+                      {
+                        "type": "SYMBOL",
+                        "name": "_kwarg_name"
+                      },
+                      {
+                        "type": "ALIAS",
+                        "content": {
+                          "type": "SYMBOL",
+                          "name": "_unexpected_keyword"
+                        },
+                        "named": true,
+                        "value": "unexpected_argument"
+                      }
+                    ]
+                  }
+                ]
               }
             ]
           }
@@ -7381,10 +7408,15 @@
       ]
     },
     "unexpected_argument": {
+      "type": "REPEAT1",
+      "content": {
+        "type": "SYMBOL",
+        "name": "_unexpected_word"
+      }
+    },
+    "_unexpected_word": {
       "type": "PREC_DYNAMIC",
       "value": -1,
-      "content": {
-        "type": "REPEAT1",
       "content": {
         "type": "SEQ",
         "members": [
@@ -7392,6 +7424,9 @@
             "type": "SYMBOL",
             "name": "_sep"
           },
+          {
+            "type": "CHOICE",
+            "members": [
               {
                 "type": "TOKEN",
                 "content": {
@@ -7402,9 +7437,72 @@
                     "value": "[^\\s%]+"
                   }
                 }
+              },
+              {
+                "type": "SEQ",
+                "members": [
+                  {
+                    "type": "ALIAS",
+                    "content": {
+                      "type": "SYMBOL",
+                      "name": "identifier"
+                    },
+                    "named": false,
+                    "value": "unexpected_word"
+                  },
+                  {
+                    "type": "CHOICE",
+                    "members": [
+                      {
+                        "type": "IMMEDIATE_TOKEN",
+                        "content": {
+                          "type": "PREC",
+                          "value": -1,
+                          "content": {
+                            "type": "PATTERN",
+                            "value": "[^\\s%]+"
+                          }
+                        }
+                      },
+                      {
+                        "type": "BLANK"
                       }
                     ]
                   }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "_unexpected_keyword": {
+      "type": "PREC_DYNAMIC",
+      "value": -1,
+      "content": {
+        "type": "SEQ",
+        "members": [
+          {
+            "type": "ALIAS",
+            "content": {
+              "type": "SYMBOL",
+              "name": "identifier"
+            },
+            "named": false,
+            "value": "unexpected_word"
+          },
+          {
+            "type": "IMMEDIATE_TOKEN",
+            "content": {
+              "type": "PREC",
+              "value": -1,
+              "content": {
+                "type": "PATTERN",
+                "value": "[^\\s%]+"
+              }
+            }
+          }
+        ]
       }
     },
     "url_tag": {
@@ -7909,6 +8007,18 @@
     [
       "unexpected_argument"
     ],
+    [
+      "variable_attribute",
+      "_unexpected_word"
+    ],
+    [
+      "resetcycle_tag",
+      "_unexpected_word"
+    ],
+    [
+      "library",
+      "_unexpected_word"
+    ],
     [
       "library",
       "load_tag"

src/node-types.json

@@ -4677,6 +4677,10 @@
     "type": "truncatewords_html",
     "named": false
   },
+  {
+    "type": "unexpected_word",
+    "named": false
+  },
   {
     "type": "unlocalize",
     "named": false

test/corpus/csp_nonce_attr.txt

@@ -66,34 +66,50 @@ Csp Nonce Attr Takes One Argument
 
 ==================
 Csp Nonce Attr Rejects Unknown Keyword
-:error
 ==================
 
 {% csp_nonce_attr other="b" %}
 
 ---
 
-(ERROR)
+(template
-
+  (content)
+  (csp_nonce_attr_tag
+    (unexpected_argument))
+  (content))
 
 ==================
 Csp Nonce Attr Rejects Repeated Keyword
-:error
 ==================
 
 {% csp_nonce_attr media=1 media=2 %}
 
 ---
 
-(ERROR)
+(template
-
+  (content)
+  (csp_nonce_attr_tag
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Csp Nonce Attr With Media Twice
-:error
 ==================
 
 {% csp_nonce_attr media="a" media="b" %}
 
 ---
 
+(template
+  (content)
+  (csp_nonce_attr_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (unexpected_argument))
+  (content))

test/corpus/custom.txt

@@ -294,13 +294,23 @@ A Name Reserved For A Tag Is Still An Argument
 
 ==================
 Custom Tag With A Keyword Argument Twice
-:error
 ==================
 
 {% mytag a=1 a=2 %}
 
 ---
 
+(template
+  (content)
+  (custom_tag
+    (identifier)
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Custom Tag With Keyword Arguments And As

test/corpus/i18n.txt

@@ -180,13 +180,21 @@ Trans Without A Message
 
 ==================
 Trans With An Option Twice
-:error
 ==================
 
 {% trans "hi" noop noop %}
 
 ---
 
+(template
+  (content)
+  (translate_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Trans With As Twice
@@ -200,13 +208,21 @@ Trans With As Twice
 
 ==================
 Trans With An Unknown Option
-:error
 ==================
 
 {% trans "hi" bogus %}
 
 ---
 
+(template
+  (content)
+  (translate_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Trans With Context And No Value

test/corpus/include.txt

@@ -163,13 +163,26 @@ Include With Only Before With
 
 ==================
 Include With A Binding Twice
-:error
 ==================
 
 {% include "t.html" with a=1 a=2 %}
 
 ---
 
+(template
+  (content)
+  (include_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Include Without A Name
@@ -193,33 +206,62 @@ Include With Nothing To Bind
 
 ==================
 Include With With Twice
-:error
 ==================
 
 {% include "t.html" with a=1 with b=2 %}
 
 ---
 
+(template
+  (content)
+  (include_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Include With Only Twice
-:error
 ==================
 
 {% include "t.html" only only %}
 
 ---
 
+(template
+  (content)
+  (include_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Include With An Unknown Option
-:error
 ==================
 
 {% include "t.html" bogus %}
 
 ---
 
+(template
+  (content)
+  (include_tag
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Include With A Legacy Binding

test/corpus/l10n.txt

@@ -95,13 +95,22 @@ Localize Nested
 
 ==================
 Localize With Another Argument
-:error
 ==================
 
 {% localize maybe %}x{% endlocalize %}
 
 ---
 
+(template
+  (content)
+  (localize_group
+    (localize_block
+      (localize_tag
+        (unexpected_argument))
+      (template
+        (content)))
+    (endlocalize_tag))
+  (content))
 
 ==================
 Localize With Two Arguments

test/corpus/querystring.txt

@@ -100,19 +100,43 @@ Querystring With Two Keys
 
 ==================
 Querystring With A Key Twice
-:error
 ==================
 
 {% querystring page=2 page=3 %}
 
 ---
 
+(template
+  (content)
+  (querystring_tag
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))
 
 ==================
 Querystring With A Key Repeated Later
-:error
 ==================
 
 {% querystring a=1 b=2 a=3 %}
 
 ---
+
+(template
+  (content)
+  (querystring_tag
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    (unexpected_argument))
+  (content))

test/corpus/static.txt

@@ -159,13 +159,17 @@ Get Media Prefix With As
 
 ==================
 Get Static Prefix Without As
-:error
 ==================
 
 {% get_static_prefix prefix %}
 
 ---
 
+(template
+  (content)
+  (get_static_prefix_tag
+    (unexpected_argument))
+  (content))
 
 ==================
 Get Static Prefix Without A Variable

test/corpus/tz.txt

@@ -54,13 +54,22 @@ Localtime Off
 
 ==================
 Localtime With Another Argument
-:error
 ==================
 
 {% localtime maybe %}x{% endlocaltime %}
 
 ---
 
+(template
+  (content)
+  (localtime_group
+    (localtime_block
+      (localtime_tag
+        (unexpected_argument))
+      (template
+        (content)))
+    (endlocaltime_tag))
+  (content))
 
 ==================
 Localtime With Two Arguments