remove min from scanner.c

grammar.js

@@ -30,6 +30,9 @@ function spaced(separator, keyword, ...args) {
   return seq(separator, keyword, ...args);
 }
 
+/** Django's string constant, which takes any escape but a line break. */
+const STRING = /"(?:[^"\\]|\\[^\n])*"|'(?:[^'\\]|\\[^\n])*'/;
+
 const django = grammar({
   name: "django",
   extras: ($) => [/[ \t]/],
@@ -72,10 +75,21 @@ const django = grammar({
     filter_expression: ($) =>
       seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
     value: ($) => choice($.literal, $.variable_attribute),
-    literal: ($) => seq(choice($.number, $.string), $._after_literal),
+    literal: ($) =>
-    number: ($) => /-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE]-?[0-9]+)?/,
+      seq(choice($.number, $.string, $.translated_string), $._after_literal),
-    string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
+    // digits, with single underscores allowed between them, as Python's
-    attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
+    // int() and float() accept ("1_000") and Django's Variable relies on
+    number: ($) =>
+      /[-+]?(?:[0-9](?:_?[0-9])*(?:\.(?:[0-9](?:_?[0-9])*)?)?|\.[0-9](?:_?[0-9])*)(?:[eE][0-9](?:_?[0-9])*)?/,
+    string: ($) => STRING,
+    /**
+     * A string to translate at render time. Django matches "_(" and ")" as
+     * part of the constant itself, so nothing may come between them and the
+     * string: "_( 'a' )" is a syntax error there, not a translated literal.
+     */
+    translated_string: ($) =>
+      seq("_(", alias(token.immediate(STRING), $.string), token.immediate(")")),
+    attribute: ($) => /[a-zA-Z0-9][a-zA-Z0-9_]*(?:\.[a-zA-Z0-9][a-zA-Z0-9_]*)*/,
     variable_attribute: ($) =>
       seq($.identifier, optional(seq(token.immediate("."), $.attribute))),
     identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,

queries/highlights.scm

@@ -34,6 +34,8 @@ variable: (identifier) @variable.parameter
     ("}}")
     ("#}")
     ("%}")
+    ("_(")
+    (")")
     (",")] @punctuation
 
 (number) @number

src/grammar.json

@@ -127,6 +127,10 @@
             {
               "type": "SYMBOL",
               "name": "string"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "translated_string"
             }
           ]
         },
@@ -138,15 +142,43 @@
     },
     "number": {
       "type": "PATTERN",
-      "value": "-?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE]-?[0-9]+)?"
+      "value": "[-+]?(?:[0-9](?:_?[0-9])*(?:\\.(?:[0-9](?:_?[0-9])*)?)?|\\.[0-9](?:_?[0-9])*)(?:[eE][0-9](?:_?[0-9])*)?"
     },
     "string": {
       "type": "PATTERN",
-      "value": "\"(?:[^\"\\\\]|\\\\\\\\|\\\\\")*\"|'(?:[^'\\\\]|\\\\\\\\|\\\\')*'"
+      "value": "\"(?:[^\"\\\\]|\\\\[^\\n])*\"|'(?:[^'\\\\]|\\\\[^\\n])*'"
+    },
+    "translated_string": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "_("
+        },
+        {
+          "type": "ALIAS",
+          "content": {
+            "type": "IMMEDIATE_TOKEN",
+            "content": {
+              "type": "PATTERN",
+              "value": "\"(?:[^\"\\\\]|\\\\[^\\n])*\"|'(?:[^'\\\\]|\\\\[^\\n])*'"
+            }
+          },
+          "named": true,
+          "value": "string"
+        },
+        {
+          "type": "IMMEDIATE_TOKEN",
+          "content": {
+            "type": "STRING",
+            "value": ")"
+          }
+        }
+      ]
     },
     "attribute": {
       "type": "PATTERN",
-      "value": "[a-zA-Z0-9_]+(?:\\.[a-zA-Z0-9_]+)*"
+      "value": "[a-zA-Z0-9][a-zA-Z0-9_]*(?:\\.[a-zA-Z0-9][a-zA-Z0-9_]*)*"
     },
     "variable_attribute": {
       "type": "SEQ",

src/node-types.json

@@ -657,6 +657,10 @@
         {
           "type": "string",
           "named": true
+        },
+        {
+          "type": "translated_string",
+          "named": true
         }
       ]
     }
@@ -1018,6 +1022,21 @@
       ]
     }
   },
+  {
+    "type": "translated_string",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "string",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "url_block",
     "named": true,
@@ -1230,6 +1249,10 @@
     "type": "%}",
     "named": false
   },
+  {
+    "type": ")",
+    "named": false
+  },
   {
     "type": ",",
     "named": false
@@ -1262,6 +1285,10 @@
     "type": ">=",
     "named": false
   },
+  {
+    "type": "_(",
+    "named": false
+  },
   {
     "type": "add:",
     "named": false

src/scanner.c

@@ -4,9 +4,7 @@
 #include <string.h>
 
 #define ERROR_SIZE 64
-#define STACK_COUNT 3
 #define NAME_SEP ' '
-#define STACK_SEP '\n'
 
 enum TokenType {
   /* indicates that an error occurred */
@@ -50,36 +48,57 @@ static const char *const separated_keywords[] = {
 #define KEYWORD_SIZE 6 // "random", the longest of them
 
 typedef Array(int32_t) Name;
-typedef Array(Name) Stack;
+
+/* The kinds of tag that open a named region. PushBlock..PushVerbatim and
+ * PopBlock..PopVerbatim are declared in this order, so a kind is the offset of
+ * its token within either run. */
+typedef enum {
+  BlockTag,
+  PartialTag,
+  VerbatimTag,
+} TagKind;
+
+/* one letter per kind, in TagKind order, used to serialize an open tag */
+static const char tag_kind_chars[] = "bpv";
+
+typedef struct {
+  TagKind kind;
+  Name name;
+} OpenTag;
+
+/* The open tags, innermost last. Only one stack is needed because the grammar
+ * nests these regions: a tag can only be closed by the tag that matches the
+ * innermost open one, so its pop token is the only one the parser ever asks
+ * for at a close tag. */
+typedef Array(OpenTag) Stack;
 
 struct Scanner {
   bool has_error;
   union {
-    Stack stacks [STACK_COUNT];
+    Stack stack;
     char error [ERROR_SIZE];
   };
 };
 
-static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
+static TagKind kind_for_push(enum TokenType token) {
-  if (token < PopBlock || token > PushVerbatim) {
+  return (TagKind) (token - PushBlock);
-    return NULL;
+}
-  }
+
-  // the pop and push tokens are in the same order, so each pair shares a stack
+static enum TokenType pop_token_for_kind(TagKind kind) {
-  return &scanner->stacks[(token - PopBlock) % STACK_COUNT];
+  return (enum TokenType) (PopBlock + kind);
 }
 
 static void reset_scanner(struct Scanner *const scanner) {
   if (scanner->has_error) {
     scanner->error[0] = '\0';
     scanner->has_error = false;
+    // the error shares storage with the stack, which is now meaningless
+    memset(&scanner->stack, 0, sizeof(scanner->stack));
   } else {
-    for (unsigned i = 0; i < STACK_COUNT; ++i) {
+    for (unsigned i = 0; i < scanner->stack.size; ++i) {
-      Stack *stack = &scanner->stacks[i];
+      array_delete(&array_get(&scanner->stack, i)->name);
-      for (unsigned i = 0; i < stack->size; ++i) {
-        array_delete(array_get(stack, i));
-      }
-      array_delete(stack);
     }
+    array_delete(&scanner->stack);
   }
 }
 
@@ -91,13 +110,6 @@ static void reset_scanner(struct Scanner *const scanner) {
   /*strncat(_scanner->error, string, ERROR_SIZE - 1);*/\
 } while(0)
 
-#define min(a, b)\
-  ({\
-    typeof(a) _a = (a);\
-    typeof(b) _b = (b);\
-    _a > _b ? _a : _b;\
-  })
-
 void * tree_sitter_django_external_scanner_create() {
   return ts_calloc(1, sizeof(struct Scanner));
 }
@@ -162,23 +174,20 @@ write_serialization_error:;
       }
     }
   } else {
-    // Ok scanner requires copying its stack
+    // Ok scanner requires copying its stack, as "<kind><name> " per open tag
-    for (unsigned i = 0; i < STACK_COUNT; ++i) {
+    for (unsigned i = 0; i < scanner->stack.size; ++i) {
-      Stack *stack = &scanner->stacks[i];
+      OpenTag *tag = array_get(&scanner->stack, i);
-      for (unsigned i = 0; i < stack->size; ++i) {
+      iter += write_code(tag_kind_chars[tag->kind], iter);
-        Name *name = array_get(stack, i);
+      for (unsigned j = 0; j < tag->name.size; ++j) {
-        for (unsigned j = 0; j < name->size; ++j) {
+        int32_t code = *array_get(&tag->name, j);
-          int32_t code = *array_get(name, j);
+        unsigned write_amt = write_code(code, iter);
-          unsigned write_amt = write_code(code, iter);
+        if (write_amt == 0) {
-          if (write_amt == 0) {
+          scanner_error(scanner, "bad code from name");
-            scanner_error(scanner, "bad code from name");
+          goto write_serialization_error;
-            goto write_serialization_error;
-          }
-          iter += write_amt;
         }
-        iter += write_code(NAME_SEP, iter);
+        iter += write_amt;
       }
-      iter += write_code(STACK_SEP, iter);
+      iter += write_code(NAME_SEP, iter);
     }
   }
   return iter - buffer;
@@ -253,11 +262,10 @@ void tree_sitter_django_external_scanner_deserialize(
     case '0': {
       scanner->has_error = false;
       const char *end = buffer + length - bytes_read;
-      Stack *stack = scanner->stacks;
+      // NULL between tags, and the name being read inside one
-      Stack *stack_end = stack + STACK_COUNT;
       Name *name = NULL;
 
-      while (stack < stack_end) {
+      while (buffer < end) {
         bytes_read = read_code(buffer, &code);
         if (bytes_read == 0) {
           scanner_error(scanner, "Bad utf8 byte");
@@ -268,34 +276,30 @@ void tree_sitter_django_external_scanner_deserialize(
           scanner_error(scanner, "Scanner truncated by length");
           return;
         }
-        if (code == STACK_SEP) {
+        if (name == NULL) {
-          ++stack;
+          // a tag starts with the letter for its kind
-          name = NULL;
+          const char *kind = code > 0 && code < 128 ? strchr(tag_kind_chars, code) : NULL;
-          continue;
+          if (kind == NULL) {
-        } else if (code == NAME_SEP) {
+            scanner_error(scanner, "Invalid tag kind");
-          if (name == NULL) {
+            return;
-            // an empty name was pushed (e.g. an anonymous verbatim block)
-            array_push(stack, (Name) array_new());
-          } else {
-            name = NULL;
           }
+          OpenTag tag = { (TagKind) (kind - tag_kind_chars), array_new() };
+          array_push(&scanner->stack, tag);
+          name = &array_back(&scanner->stack)->name;
+        } else if (code == NAME_SEP) {
+          // an empty name means an unnamed tag (e.g. an anonymous verbatim)
+          name = NULL;
         } else {
-          if (name == NULL) {
+          bool valid = name->size ? check_name_char(code) : check_name_start_char(code);
-            if (!check_name_start_char(code)) {
+          if (!valid) {
-              scanner_error(scanner, "Invalid name, must start with letter");
-              return;
-            }
-            array_push(stack, (Name) array_new());
-            name = array_back(stack);
-          } else if (!check_name_char(code)) {
             scanner_error(scanner, "Invalid character in name");
             return;
           }
           array_push(name, code);
         }
       }
-      if (buffer != end) {
+      if (name != NULL) {
-        scanner_error(scanner, "Scanner deserialization finished with left over length");
+        scanner_error(scanner, "Scanner deserialization finished mid-name");
       }
     }
   }
@@ -337,6 +341,12 @@ static bool check_close_block(TSLexer *const lexer) {
   return lexer->lookahead == '}';
 }
 
+/* Marks the name just read as the token, then checks that only "%}" follows. */
+static bool check_close_block_from(TSLexer *const lexer) {
+  lexer->mark_end(lexer);
+  return check_close_block(lexer);
+}
+
 const char inline_chars[] = "inline";
 
 static bool check_inline(TSLexer *const lexer) {
@@ -396,11 +406,14 @@ static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name
  * unrelated (e.g. unnamed) "{% verbatim %}...{% endverbatim %}" pair appear
  * unparsed inside a named verbatim block. */
 static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const lexer) {
-  Stack *stack = get_stack_for_token(scanner, PopVerbatim);
+  if (scanner->stack.size == 0) {
-  if (stack->size == 0) {
     return false;
   }
-  Name *name = array_back(stack);
+  OpenTag *tag = array_back(&scanner->stack);
+  if (tag->kind != VerbatimTag) {
+    return false;
+  }
+  Name *name = &tag->name;
   bool consumed_any = false;
   while (true) {
     if (lexer->eof(lexer)) {
@@ -451,7 +464,9 @@ static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const
  * it, which leaves this the only place to catch the run-on. */
 static bool scan_after_literal(TSLexer *const lexer) {
   int32_t next = lexer->lookahead;
-  if (check_name_char(next) || next == '\'' || next == '"' || next == '-') {
+  // anything that could begin another value: a name, a quote, or a sign or
+  // point that would start a second number ("{% cycle 1.2.3 %}")
+  if (check_name_char(next) || next == '\'' || next == '"' || next == '-' || next == '+' || next == '.') {
     return false;
   }
   lexer->mark_end(lexer);
@@ -603,68 +618,49 @@ bool tree_sitter_django_external_scanner_scan(
     is_empty = true;
   }
 
-  // save most matched name in case we need to reread characters
+  /* Only the innermost open tag can be closed here, so it is the one and only
-  Name *most_matched_name;
+   * candidate: there is never a second name to try, and so never any input to
-  unsigned most_matched_amt = 0;
+   * reread. */
-  for (unsigned token = PopBlock; token < PushBlock; ++token) {
+  if (scanner->stack.size > 0) {
-    Stack *stack = get_stack_for_token(scanner, token);
+    OpenTag *tag = array_back(&scanner->stack);
-    if (valid_symbols[token] && stack->size > 0) {
+    enum TokenType token = pop_token_for_kind(tag->kind);
-      Name *name = array_back(stack);
+    if (valid_symbols[token]) {
       if (is_empty) {
         // we matched a close block without an id
-        array_delete(name);
+        array_delete(&tag->name);
-        array_pop(stack);
+        array_pop(&scanner->stack);
         lexer->result_symbol = token;
         return true;
       }
-      if (most_matched_amt > name->size) {
+      unsigned match_amt = check_name(lexer, &tag->name);
-        continue;
+      bool has_next_char = tag->name.size ? check_name_char(lexer->lookahead) : check_name_start_char(lexer->lookahead);
+      if (match_amt == tag->name.size && !has_next_char && check_close_block_from(lexer)) {
+        array_delete(&tag->name);
+        array_pop(&scanner->stack);
+        lexer->result_symbol = token;
+        return true;
       }
-      if (most_matched_amt > 0) {
+      // the close tag names something other than the tag it would close
-        for (int i = 0; i < most_matched_amt; ++i) {
+      return false;
-          if (*array_get(most_matched_name, i) != *array_get(name, i)) {
-            continue;
-          }
-        }
-      }
-      unsigned match_amt = check_name(lexer, name) + most_matched_amt;
-      bool has_next_char = name->size ? check_name_char(lexer->lookahead) : check_name_start_char(lexer->lookahead);
-      if (match_amt == name->size && !has_next_char) {
-        // name matches and there are no remaining name chars from lexer
-        lexer->mark_end(lexer);
-        if (check_close_block(lexer)) {
-          array_delete(name);
-          array_pop(stack);
-          lexer->result_symbol = token;
-          return true;
-        } else {
-          // bad close block means no matches
-          return false;
-        }
-      }
-      most_matched_amt = match_amt;
-      most_matched_name = name;
     }
   }
   for (unsigned token = PushBlock; token <= PushVerbatim; ++token) {
     if (valid_symbols[token]) {
-      Stack *stack = get_stack_for_token(scanner, token);
+      TagKind kind = kind_for_push(token);
       if (is_empty && token == PushVerbatim) {
         // use empty string for name to indicate empty push
-        array_push(stack, (Name) array_new());
+        OpenTag tag = { kind, array_new() };
+        array_push(&scanner->stack, tag);
         lexer->result_symbol = token;
         return true;
       }
-      // do not push stack until we are done with most_matched_name
       Name name = array_new();
-      if (most_matched_amt > 0) {
-        array_extend(&name, most_matched_amt, most_matched_name);
-      }
       if (read_name(lexer, &name)) {
         // validate tokens after name
         lexer->mark_end(lexer);
         if (check_close_block(lexer) || token == PushPartial && check_inline(lexer)) {
-          array_push(stack, name);
+          OpenTag tag = { kind, name };
+          array_push(&scanner->stack, tag);
           lexer->result_symbol = token;
           return true;
         }

test/corpus/cycle.txt

@@ -113,3 +113,14 @@ Cycle Value Adjacent To Value
 ---
 
 (ERROR)
+
+==================
+Cycle Number Run On To Number
+:error
+==================
+
+{% cycle 1.2.3 %}
+
+---
+
+(ERROR)

test/corpus/template_variable.txt

@@ -211,3 +211,115 @@ Template Variable Literal Adjacent To Name
 ---
 
 (ERROR)
+
+==================
+Template Variable With Escaped String
+==================
+
+{{ "a\nb\"c" }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Template Variable Signed Number
+==================
+
+{{ +1.5 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Template Variable Underscore Separated Number
+==================
+
+{{ 1_000 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Template Variable Attribute Beginning With Underscore
+:error
+==================
+
+{{ user._private }}
+
+---
+
+(ERROR)
+
+==================
+Template Variable Translated String
+==================
+
+{{ _("Hello") }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (translated_string
+            (string))))))
+  (content))
+
+==================
+Template Variable Translated String With Filter
+==================
+
+{{ _('Hello')|upper }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (translated_string
+            (string))))
+      (filter_expression
+        (filter))))
+  (content))
+
+==================
+Template Variable Translated String With Spaces
+:error
+==================
+
+{{ _( "Hello" ) }}
+
+---
+
+(ERROR)
+