fixed whitespace issue

grammar.js

@@ -11,12 +11,33 @@ function block(tag, ...args) {
   return seq("{%", field("tag", tag), ...args, "%}");
 }
 
+/**
+ * A keyword that must be separated from the token before it by whitespace.
+ * Django splits tag contents on whitespace, so "{% cycle 1as x %}" is one
+ * token and an error, not "1" followed by "as". Nothing in the lexer enforces
+ * that on its own: two keywords cannot run together (the lexer would read one
+ * longer identifier), but a literal can run into the keyword after it, and
+ * extras can never be made mandatory. The separator is therefore a zero-width
+ * external token that the scanner only emits when whitespace really precedes
+ * its keyword.
+ *
+ * @param {RuleOrLiteral} separator
+ * @param {string} keyword
+ * @param {...RuleOrLiteral} args
+ * @returns {SeqRule}
+ */
+function spaced(separator, keyword, ...args) {
+  return seq(separator, keyword, ...args);
+}
+
 const django = grammar({
   name: "django",
   extras: ($) => [/[ \t]/],
+  word: ($) => $.identifier,
   conflicts: ($) => [[$.template]],
   supertypes: ($) => [$.template_tag, $.template_block_groups],
   externals: ($) => [
+    $.matcher_error,
     $.pop_block,
     $.pop_partial,
     $.pop_verbatim,
@@ -25,7 +46,18 @@ const django = grammar({
     $.push_verbatim,
     $.verbatim_content,
     $.comment_content,
-    $.matcher_error,
+    $._before_and,
+    $._before_as,
+    $._before_b,
+    $._before_by,
+    $._before_in,
+    $._before_is,
+    $._before_not,
+    $._before_or,
+    $._before_p,
+    $._before_random,
+    $._before_w,
+    $._after_literal,
   ],
   reserved: {
     global: ($) => ["not", "if", "in", "is", "as", "for", "from"],
@@ -40,7 +72,7 @@ const django = grammar({
     filter_expression: ($) =>
       seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
     value: ($) => choice($.literal, $.variable_attribute),
-    literal: ($) => choice($.number, $.string),
+    literal: ($) => seq(choice($.number, $.string), $._after_literal),
     number: ($) => /-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE]-?[0-9]+)?/,
     string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
     attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
@@ -49,24 +81,30 @@ const django = grammar({
     identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,
     binaryOperator: ($) =>
       choice(
-        "and",
+        spaced($._before_and, "and"),
-        "or",
+        spaced($._before_or, "or"),
         "==",
         "!=",
         "<",
         ">",
         "<=",
         ">=",
-        "in",
+        spaced($._before_in, "in"),
-        prec(1, seq("not", "in")),
+        prec(1, spaced($._before_not, "not", "in")),
-        "is",
+        spaced($._before_is, "is"),
-        prec(1, seq("is", "not")),
+        prec(1, spaced($._before_is, "is", "not")),
       ),
     predicate: ($) =>
       seq(
-        optional("not"),
+        optional(spaced($._before_not, "not")),
         $.filtered_value,
-        repeat(seq($.binaryOperator, optional("not"), $.filtered_value)),
+        repeat(
+          seq(
+            $.binaryOperator,
+            optional(spaced($._before_not, "not")),
+            $.filtered_value,
+          ),
+        ),
       ),
     template_variable: ($) => seq("{{", $.filtered_value, "}}"),
     template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"),
@@ -183,9 +221,17 @@ const django = grammar({
       block(
         "cycle",
         repeat1($.filtered_value),
-        optional(seq("as", field("variable", $.identifier))),
+        optional(
+          spaced(
+            $._before_as,
+            "as",
+            field("variable", $.identifier),
+            // only the as-form takes the flag, and an identifier can never run
+            // into it, so "silent" needs no separator of its own
             optional("silent"),
           ),
+        ),
+      ),
     debug: ($) => block("debug"),
     extends: ($) => block("extends", $.filtered_value),
     filter_group: ($) =>
@@ -198,7 +244,7 @@ const django = grammar({
       block(
         "firstof",
         repeat1($.filtered_value),
-        optional(seq("as", $.identifier)),
+        optional(spaced($._before_as, "as", $.identifier)),
       ),
     for_scope: ($) =>
       prec.left(
@@ -251,11 +297,19 @@ const django = grammar({
       block(
         "lorem",
         $.filtered_value,
-        choice("w", "p", "b"),
+        choice(
-        optional("random"),
+          spaced($._before_w, "w"),
+          spaced($._before_p, "p"),
+          spaced($._before_b, "b"),
+        ),
+        optional(spaced($._before_random, "random")),
       ),
     now: ($) =>
-      block("now", $.string, optional(seq("as", field("name", $.identifier)))),
+      block(
+        "now",
+        $.string,
+        optional(spaced($._before_as, "as", field("name", $.identifier))),
+      ),
     partial: ($) => block("partial", $.identifier),
     partialdef_group: ($) =>
       seq(
@@ -273,9 +327,9 @@ const django = grammar({
       block(
         "regroup",
         $.filtered_value,
-        "by",
+        spaced($._before_by, "by"),
         $.attribute,
-        optional(seq("as", field("variable", $.identifier))),
+        optional(spaced($._before_as, "as", field("variable", $.identifier))),
       ),
     reset_cycle: ($) => block("resetcycle", optional($.identifier)),
     spaceless_group: ($) =>
@@ -304,7 +358,7 @@ const django = grammar({
             repeat1(seq($.identifier, "=", $.filtered_value)),
           ),
         ),
-        optional(seq("as", field("variable", $.identifier))),
+        optional(spaced($._before_as, "as", field("variable", $.identifier))),
       ),
     verbatim_group: ($) =>
       seq(
@@ -318,7 +372,7 @@ const django = grammar({
         $.filtered_value,
         $.filtered_value,
         $.filtered_value,
-        optional(seq("as", field("variable", $.identifier))),
+        optional(spaced($._before_as, "as", field("variable", $.identifier))),
       ),
     with_group: ($) =>
       seq(

src/grammar.json

@@ -1,6 +1,7 @@
 {
   "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
   "name": "django",
+  "word": "identifier",
   "rules": {
     "template": {
       "type": "REPEAT1",
@@ -114,6 +115,9 @@
       ]
     },
     "literal": {
+      "type": "SEQ",
+      "members": [
+        {
           "type": "CHOICE",
           "members": [
             {
@@ -126,6 +130,12 @@
             }
           ]
         },
+        {
+          "type": "SYMBOL",
+          "name": "_after_literal"
+        }
+      ]
+    },
     "number": {
       "type": "PATTERN",
       "value": "-?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE]-?[0-9]+)?"
@@ -178,13 +188,31 @@
     "binaryOperator": {
       "type": "CHOICE",
       "members": [
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_before_and"
+            },
             {
               "type": "STRING",
               "value": "and"
+            }
+          ]
+        },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_before_or"
             },
             {
               "type": "STRING",
               "value": "or"
+            }
+          ]
         },
         {
           "type": "STRING",
@@ -210,9 +238,18 @@
           "type": "STRING",
           "value": ">="
         },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_before_in"
+            },
             {
               "type": "STRING",
               "value": "in"
+            }
+          ]
         },
         {
           "type": "PREC",
@@ -220,6 +257,10 @@
           "content": {
             "type": "SEQ",
             "members": [
+              {
+                "type": "SYMBOL",
+                "name": "_before_not"
+              },
               {
                 "type": "STRING",
                 "value": "not"
@@ -231,9 +272,18 @@
             ]
           }
         },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_before_is"
+            },
             {
               "type": "STRING",
               "value": "is"
+            }
+          ]
         },
         {
           "type": "PREC",
@@ -241,6 +291,10 @@
           "content": {
             "type": "SEQ",
             "members": [
+              {
+                "type": "SYMBOL",
+                "name": "_before_is"
+              },
               {
                 "type": "STRING",
                 "value": "is"
@@ -260,9 +314,18 @@
         {
           "type": "CHOICE",
           "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_not"
+                },
                 {
                   "type": "STRING",
                   "value": "not"
+                }
+              ]
             },
             {
               "type": "BLANK"
@@ -285,9 +348,18 @@
               {
                 "type": "CHOICE",
                 "members": [
+                  {
+                    "type": "SEQ",
+                    "members": [
+                      {
+                        "type": "SYMBOL",
+                        "name": "_before_not"
+                      },
                       {
                         "type": "STRING",
                         "value": "not"
+                      }
+                    ]
                   },
                   {
                     "type": "BLANK"
@@ -1297,6 +1369,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -1308,13 +1384,6 @@
                     "type": "SYMBOL",
                     "name": "identifier"
                   }
-                }
-              ]
-            },
-            {
-              "type": "BLANK"
-            }
-          ]
                 },
                 {
                   "type": "CHOICE",
@@ -1327,6 +1396,13 @@
                       "type": "BLANK"
                     }
                   ]
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         },
         {
           "type": "STRING",
@@ -1471,6 +1547,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -2003,26 +2083,62 @@
         {
           "type": "CHOICE",
           "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_w"
+                },
                 {
                   "type": "STRING",
                   "value": "w"
+                }
+              ]
+            },
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_p"
                 },
                 {
                   "type": "STRING",
                   "value": "p"
+                }
+              ]
+            },
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_b"
                 },
                 {
                   "type": "STRING",
                   "value": "b"
                 }
               ]
+            }
+          ]
         },
         {
           "type": "CHOICE",
           "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_random"
+                },
                 {
                   "type": "STRING",
                   "value": "random"
+                }
+              ]
             },
             {
               "type": "BLANK"
@@ -2060,6 +2176,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -2256,9 +2376,18 @@
           "type": "SYMBOL",
           "name": "filtered_value"
         },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_before_by"
+            },
             {
               "type": "STRING",
               "value": "by"
+            }
+          ]
         },
         {
           "type": "SYMBOL",
@@ -2270,6 +2399,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -2519,6 +2652,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -2644,6 +2781,10 @@
             {
               "type": "SEQ",
               "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "_before_as"
+                },
                 {
                   "type": "STRING",
                   "value": "as"
@@ -2766,6 +2907,10 @@
   ],
   "precedences": [],
   "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "matcher_error"
+    },
     {
       "type": "SYMBOL",
       "name": "pop_block"
@@ -2800,7 +2945,51 @@
     },
     {
       "type": "SYMBOL",
-      "name": "matcher_error"
+      "name": "_before_and"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_as"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_b"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_by"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_in"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_is"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_not"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_or"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_p"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_random"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_before_w"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_after_literal"
     }
   ],
   "inline": [],

src/scanner.c

@@ -1,6 +1,7 @@
 #include "tree_sitter/parser.h"
 #include "tree_sitter/alloc.h"
 #include "tree_sitter/array.h"
+#include <string.h>
 
 #define ERROR_SIZE 64
 #define STACK_COUNT 3
@@ -8,6 +9,8 @@
 #define STACK_SEP '\n'
 
 enum TokenType {
+  /* indicates that an error occurred */
+  MatcherError,
   /* push name on stack to match with later */
   PopBlock,     // [ name ] %}
   PopPartial,   // [ name ] %}
@@ -20,10 +23,32 @@ enum TokenType {
   VerbatimContent,
   /* raw text inside a comment block, up through (but excluding) the endcomment tag */
   CommentContent,
-  /* indicates that an error occurred */
+  /* zero-width markers for the whitespace preceding a keyword, in the same
-  MatcherError,
+   * order as separated_keywords below */
+  BeforeAnd,
+  BeforeAs,
+  BeforeB,
+  BeforeBy,
+  BeforeIn,
+  BeforeIs,
+  BeforeNot,
+  BeforeOr,
+  BeforeP,
+  BeforeRandom,
+  BeforeW,
+  /* zero-width marker for the end of a literal */
+  AfterLiteral,
 };
 
+/* Keywords Django only reads as keywords when whitespace separates them from
+ * the token before, listed in BeforeAnd..BeforeW order. */
+static const char *const separated_keywords[] = {
+  "and", "as", "b", "by", "in", "is", "not", "or", "p", "random", "w",
+};
+
+#define KEYWORD_COUNT (sizeof(separated_keywords) / sizeof(*separated_keywords))
+#define KEYWORD_SIZE 6 // "random", the longest of them
+
 typedef Array(int32_t) Name;
 typedef Array(Name) Stack;
 
@@ -36,10 +61,11 @@ struct Scanner {
 };
 
 static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
-  if (token >= MatcherError) {
+  if (token < PopBlock || token > PushVerbatim) {
     return NULL;
   }
-  return &scanner->stacks[token % STACK_COUNT];
+  // the pop and push tokens are in the same order, so each pair shares a stack
+  return &scanner->stacks[(token - PopBlock) % STACK_COUNT];
 }
 
 static void reset_scanner(struct Scanner *const scanner) {
@@ -406,6 +432,66 @@ static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const
   }
 }
 
+/* Emits a zero-width token at the whitespace that separates one of the
+ * separated_keywords from the token before it. Django splits tag contents on
+ * whitespace, so those words are only keywords when something separates them;
+ * without this, "{% cycle 1as x %}" would parse as a cycle over 1 bound to x.
+ * Two keywords cannot run together (the lexer reads one longer identifier
+ * instead), but a literal can run into the keyword after it, and the grammar's
+ * extras can never be made mandatory.
+ *
+ * Reads the whole word that follows and only fires when it is a keyword the
+ * parser is currently expecting, since the whitespace after a value is equally
+ * the whitespace before another value ("{% cycle a b %}"). */
+/* Emits a zero-width token at the end of a literal, and fails when the
+ * literal runs straight into whatever follows it. Django splits tag contents
+ * on whitespace, so "{% cycle 1a %}" is the single token "1a" and an error,
+ * not the number 1 followed by the variable a. A literal ends wherever its
+ * own pattern ends, so unlike an identifier it cannot swallow the text after
+ * it, which leaves this the only place to catch the run-on. */
+static bool scan_after_literal(TSLexer *const lexer) {
+  int32_t next = lexer->lookahead;
+  if (check_name_char(next) || next == '\'' || next == '"' || next == '-') {
+    return false;
+  }
+  lexer->mark_end(lexer);
+  lexer->result_symbol = AfterLiteral;
+  return true;
+}
+
+static bool scan_separator(TSLexer *const lexer, const bool *const valid_symbols) {
+  bool separated = false;
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
+    lexer->advance(lexer, true);
+    separated = true;
+  }
+  if (!separated) {
+    return false;
+  }
+  // the token ends here, before the keyword itself, which the grammar lexes
+  lexer->mark_end(lexer);
+
+  char word[KEYWORD_SIZE + 1];
+  unsigned length = 0;
+  while (check_name_char(lexer->lookahead)) {
+    if (length == KEYWORD_SIZE) {
+      // too long to be one of them, and a prefix match would be wrong anyway
+      return false;
+    }
+    word[length++] = (char) lexer->lookahead;
+    lexer->advance(lexer, false);
+  }
+  word[length] = '\0';
+
+  for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
+    if (valid_symbols[BeforeAnd + i] && strcmp(word, separated_keywords[i]) == 0) {
+      lexer->result_symbol = BeforeAnd + i;
+      return true;
+    }
+  }
+  return false;
+}
+
 const char endcomment_chars[] = "endcomment";
 
 /* Called right after consuming "{%"; checks whether what follows closes the
@@ -486,6 +572,24 @@ bool tree_sitter_django_external_scanner_scan(
   if (valid_symbols[CommentContent]) {
     return scan_comment_content(lexer);
   }
+  if (valid_symbols[AfterLiteral]) {
+    return scan_after_literal(lexer);
+  }
+  bool separator = false;
+  for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
+    separator |= valid_symbols[BeforeAnd + i];
+  }
+  if (separator) {
+    bool matching_name = false;
+    for (unsigned token = PopBlock; token <= PushVerbatim; ++token) {
+      matching_name |= valid_symbols[token];
+    }
+    /* these keywords never follow a tag name that opens or closes a named
+     * block, so the two scans never compete over the same position */
+    if (!matching_name) {
+      return scan_separator(lexer, valid_symbols);
+    }
+  }
   while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
     lexer->advance(lexer, true);
   }
@@ -542,7 +646,7 @@ bool tree_sitter_django_external_scanner_scan(
       most_matched_name = name;
     }
   }
-  for (unsigned token = PushBlock; token < MatcherError; ++token) {
+  for (unsigned token = PushBlock; token <= PushVerbatim; ++token) {
     if (valid_symbols[token]) {
       Stack *stack = get_stack_for_token(scanner, token);
       if (is_empty && token == PushVerbatim) {

test/corpus/cycle.txt

@@ -91,3 +91,25 @@ Cycle Referencing Existing Variable
         (variable_attribute
           (identifier)))))
   (content))
+
+==================
+Cycle With As Adjacent To Value
+:error
+==================
+
+{% cycle 1 2as rowcolors %}
+
+---
+
+(ERROR)
+
+==================
+Cycle Value Adjacent To Value
+:error
+==================
+
+{% cycle 'row1'rowcolors %}
+
+---
+
+(ERROR)

test/corpus/if.txt

@@ -141,3 +141,14 @@ yes
     (template
       (content)))
   (content))
+
+==================
+If Operator Adjacent To Value
+:error
+==================
+
+{% if 1and 2 %}yes{% endif %}
+
+---
+
+(ERROR)

test/corpus/lorem.txt

@@ -31,3 +31,14 @@ Lorem Paragraphs Random
         (literal
           (number)))))
   (content))
+
+==================
+Lorem Method Adjacent To Count
+:error
+==================
+
+{% lorem 5w %}
+
+---
+
+(ERROR)

test/corpus/regroup.txt

@@ -16,3 +16,14 @@ Regroup Basic
     (attribute)
     variable: (identifier))
   (content))
+
+==================
+Regroup By Adjacent To Value
+:error
+==================
+
+{% regroup 1by gender as grouped %}
+
+---
+
+(ERROR)

test/corpus/template_variable.txt

@@ -200,3 +200,14 @@ Template Variable Number With Filter Argument
             (literal
               (number)))))))
   (content))
+
+==================
+Template Variable Literal Adjacent To Name
+:error
+==================
+
+{{ 1x }}
+
+---
+
+(ERROR)