Support new Function parsing in the pre-scanner. (#3110)

JerryScript-DCO-1.0-Signed-off-by: Zoltan Herczeg zherczeg.u-szeged@partner.samsung.com

jerry-core/parser/js/js-parser-internal.h

@@ -572,7 +572,8 @@ void scanner_cleanup (parser_context_t *context_p);
 void scanner_get_location (scanner_location_t *location_p, parser_context_t *context_p);
 void scanner_set_location (parser_context_t *context_p, scanner_location_t *location_p);
 
-void scanner_scan_all (parser_context_t *context_p);
+void scanner_scan_all (parser_context_t *context_p, const uint8_t *arg_list_p, const uint8_t *arg_list_end_p,
+                       const uint8_t *source_p, const uint8_t *source_end_p);
 
 /**
  * @}

jerry-core/parser/js/js-parser.c

@@ -2465,12 +2465,11 @@ parser_parse_source (const uint8_t *arg_list_p, /**< function argument list */
   }
 #endif /* ENABLED (JERRY_PARSER_DUMP_BYTE_CODE) */
 
-  context.source_p = source_p;
+  scanner_scan_all (&context,
-  context.source_end_p = source_p + source_size;
+                    arg_list_p,
-  context.line = 1;
+                    arg_list_p + arg_list_size,
-  context.column = 1;
+                    source_p,
-
+                    source_p + source_size);
-  scanner_scan_all (&context);
 
   if (JERRY_UNLIKELY (context.error != PARSER_ERR_NO_ERROR))
   {
@@ -2519,6 +2518,9 @@ parser_parse_source (const uint8_t *arg_list_p, /**< function argument list */
     {
       parser_parse_function_arguments (&context, LEXER_EOS);
 
+      JERRY_ASSERT (context.next_scanner_info_p->type == SCANNER_TYPE_END_ARGUMENTS);
+      scanner_release_next (&context, sizeof (scanner_info_t));
+
       context.source_p = source_p;
       context.source_end_p = source_p + source_size;
       context.line = 1;

jerry-core/parser/js/js-scanner-util.c

@@ -242,9 +242,10 @@ scanner_cleanup (parser_context_t *context_p) /**< context */
       default:
       {
 #if ENABLED (JERRY_ES2015_ARROW_FUNCTION)
-        JERRY_ASSERT (scanner_info_p->type == SCANNER_TYPE_ARROW);
+        JERRY_ASSERT (scanner_info_p->type == SCANNER_TYPE_END_ARGUMENTS
+                      || scanner_info_p->type == SCANNER_TYPE_ARROW);
 #else /* !ENABLED (JERRY_ES2015_ARROW_FUNCTION) */
-        JERRY_ASSERT (0);
+        JERRY_ASSERT (scanner_info_p->type == SCANNER_TYPE_END_ARGUMENTS);
 #endif /* ENABLED (JERRY_ES2015_ARROW_FUNCTION) */
         break;
       }

jerry-core/parser/js/js-scanner.c

@@ -59,7 +59,8 @@ typedef enum
  */
 typedef enum
 {
-  SCAN_STACK_HEAD,                         /**< head */
+  SCAN_STACK_SCRIPT,                       /**< script */
+  SCAN_STACK_EVAL_FUNCTION,                /**< evaluated function */
   SCAN_STACK_BLOCK_STATEMENT,              /**< block statement group */
   SCAN_STACK_FUNCTION_STATEMENT,           /**< function statement */
   SCAN_STACK_FUNCTION_EXPRESSION,          /**< function expression */
@@ -454,14 +455,18 @@ scanner_scan_primary_expression_end (parser_context_t *context_p, /**< context *
     case SCAN_STACK_VAR:
     {
       parser_stack_pop_uint8 (context_p);
-
+      /* FALLTHRU */
+    }
+    case SCAN_STACK_SCRIPT:
+    case SCAN_STACK_EVAL_FUNCTION:
+    {
       if (type == LEXER_EOS)
       {
+        scanner_context_p->mode = SCAN_MODE_STATEMENT;
         return true;
       }
       /* FALLTHRU */
     }
-    case SCAN_STACK_HEAD:
     case SCAN_STACK_BLOCK_STATEMENT:
     case SCAN_STACK_FUNCTION_STATEMENT:
     case SCAN_STACK_FUNCTION_EXPRESSION:
@@ -780,13 +785,15 @@ scanner_scan_primary_expression_end (parser_context_t *context_p, /**< context *
 #if ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER)
     case SCAN_STACK_FUNCTION_PARAMETERS:
     {
-      if (type != LEXER_RIGHT_PAREN)
+      parser_stack_pop_uint8 (context_p);
+
+      if (type != LEXER_RIGHT_PAREN
+          && (type != LEXER_EOS || context_p->stack_top_uint8 != SCAN_STACK_EVAL_FUNCTION))
       {
         break;
       }
 
       scanner_context_p->mode = SCAN_MODE_CONTINUE_FUNCTION_ARGUMENTS;
-      parser_stack_pop_uint8 (context_p);
       return true;
     }
 #endif /* ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER) */
@@ -1070,7 +1077,7 @@ scanner_scan_statement (parser_context_t *context_p, /**< context */
 #if ENABLED (JERRY_ES2015_MODULE_SYSTEM)
     case LEXER_KEYW_IMPORT:
     {
-      if (stack_top != SCAN_STACK_HEAD)
+      if (stack_top != SCAN_STACK_SCRIPT)
       {
         scanner_raise_error (context_p);
       }
@@ -1183,7 +1190,7 @@ scanner_scan_statement (parser_context_t *context_p, /**< context */
     }
     case LEXER_KEYW_EXPORT:
     {
-      if (stack_top != SCAN_STACK_HEAD)
+      if (stack_top != SCAN_STACK_SCRIPT)
       {
         scanner_raise_error (context_p);
       }
@@ -1327,47 +1334,56 @@ scanner_scan_statement (parser_context_t *context_p, /**< context */
  * Scan the whole source code.
  */
 void
-scanner_scan_all (parser_context_t *context_p) /**< context */
+scanner_scan_all (parser_context_t *context_p, /**< context */
+                  const uint8_t *arg_list_p, /**< function argument list */
+                  const uint8_t *arg_list_end_p, /**< end of argument list */
+                  const uint8_t *source_p, /**< valid UTF-8 source code */
+                  const uint8_t *source_end_p) /**< end of source code */
 {
   scanner_context_t scanner_context;
 
 #if ENABLED (JERRY_PARSER_DUMP_BYTE_CODE)
-  const uint8_t *source_start_p = context_p->source_p;
-
   if (context_p->is_show_opcodes)
   {
     JERRY_DEBUG_MSG ("\n--- Scanning start ---\n\n");
   }
 #endif /* ENABLED (JERRY_PARSER_DUMP_BYTE_CODE) */
 
-  scanner_context.mode = SCAN_MODE_STATEMENT;
   scanner_context.active_switch_statement.last_case_p = NULL;
 
   parser_stack_init (context_p);
 
   PARSER_TRY (context_p->try_buffer)
   {
-    parser_stack_push_uint8 (context_p, SCAN_STACK_HEAD);
+    context_p->line = 1;
-    lexer_next_token (context_p);
+    context_p->column = 1;
+
+    if (arg_list_p == NULL)
+    {
+      context_p->source_p = source_p;
+      context_p->source_end_p = source_end_p;
+
+      scanner_context.mode = SCAN_MODE_STATEMENT;
+      parser_stack_push_uint8 (context_p, SCAN_STACK_SCRIPT);
+
+      lexer_next_token (context_p);
+    }
+    else
+    {
+      context_p->source_p = arg_list_p;
+      context_p->source_end_p = arg_list_end_p;
+      scanner_context.mode = SCAN_MODE_FUNCTION_ARGUMENTS;
+      parser_stack_push_uint8 (context_p, SCAN_STACK_EVAL_FUNCTION);
+
+      /* Faking the first token. */
+      context_p->token.type = LEXER_LEFT_PAREN;
+    }
 
     while (true)
     {
       lexer_token_type_t type = (lexer_token_type_t) context_p->token.type;
       scan_stack_modes_t stack_top = (scan_stack_modes_t) context_p->stack_top_uint8;
 
-      if (type == LEXER_EOS)
-      {
-#ifndef JERRY_NDEBUG
-        context_p->status_flags |= PARSER_SCANNING_SUCCESSFUL;
-#endif /* !JERRY_NDEBUG */
-
-        if (stack_top == SCAN_STACK_HEAD)
-        {
-          break;
-        }
-        scanner_raise_error (context_p);
-      }
-
       switch (scanner_context.mode)
       {
         case SCAN_MODE_PRIMARY_EXPRESSION:
@@ -1498,6 +1514,11 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
         }
         case SCAN_MODE_STATEMENT:
         {
+          if (type == LEXER_EOS)
+          {
+            goto scan_completed;
+          }
+
           if (scanner_scan_statement (context_p, &scanner_context, type, stack_top))
           {
             continue;
@@ -1559,7 +1580,8 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
         }
         case SCAN_MODE_FUNCTION_ARGUMENTS:
         {
-          JERRY_ASSERT (stack_top == SCAN_STACK_FUNCTION_STATEMENT
+          JERRY_ASSERT (stack_top == SCAN_STACK_EVAL_FUNCTION
+                        || stack_top == SCAN_STACK_FUNCTION_STATEMENT
                         || stack_top == SCAN_STACK_FUNCTION_EXPRESSION
                         || stack_top == SCAN_STACK_FUNCTION_PROPERTY);
 
@@ -1576,7 +1598,7 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
         {
 #endif /* ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER) */
 
-          if (context_p->token.type != LEXER_RIGHT_PAREN)
+          if (context_p->token.type != LEXER_RIGHT_PAREN && context_p->token.type != LEXER_EOS)
           {
             while (true)
             {
@@ -1611,6 +1633,25 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
           }
 #endif /* ENABLED (JERRY_ES2015_FUNCTION_PARAMETER_INITIALIZER) */
 
+          if (context_p->token.type == LEXER_EOS && stack_top == SCAN_STACK_EVAL_FUNCTION)
+          {
+            /* End of argument parsing. */
+            scanner_info_t *scanner_info_p = (scanner_info_t *) scanner_malloc (context_p, sizeof (scanner_info_t));
+            scanner_info_p->next_p = context_p->next_scanner_info_p;
+            scanner_info_p->source_p = NULL;
+            scanner_info_p->type = SCANNER_TYPE_END_ARGUMENTS;
+
+            context_p->next_scanner_info_p = scanner_info_p;
+            context_p->source_p = source_p;
+            context_p->source_end_p = source_end_p;
+            context_p->line = 1;
+            context_p->column = 1;
+
+            scanner_context.mode = SCAN_MODE_STATEMENT;
+            lexer_next_token (context_p);
+            continue;
+          }
+
           if (context_p->token.type != LEXER_RIGHT_PAREN)
           {
             scanner_raise_error (context_p);
@@ -1707,6 +1748,17 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
 
       lexer_next_token (context_p);
     }
+
+scan_completed:
+    if (context_p->stack_top_uint8 != SCAN_STACK_SCRIPT
+        && context_p->stack_top_uint8 != SCAN_STACK_EVAL_FUNCTION)
+    {
+      scanner_raise_error (context_p);
+    }
+
+#ifndef JERRY_NDEBUG
+    context_p->status_flags |= PARSER_SCANNING_SUCCESSFUL;
+#endif /* !JERRY_NDEBUG */
   }
   PARSER_CATCH
   {
@@ -1724,6 +1776,7 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
   if (context_p->is_show_opcodes)
   {
     scanner_info_t *info_p = context_p->next_scanner_info_p;
+    const uint8_t *source_start_p = (arg_list_p == NULL) ? source_p : arg_list_p;
 
     while (info_p->type != SCANNER_TYPE_END)
     {
@@ -1732,6 +1785,12 @@ scanner_scan_all (parser_context_t *context_p) /**< context */
 
       switch (info_p->type)
       {
+        case SCANNER_TYPE_END_ARGUMENTS:
+        {
+          JERRY_DEBUG_MSG ("  END_ARGUMENTS\n");
+          source_start_p = source_p;
+          break;
+        }
         case SCANNER_TYPE_WHILE:
         {
           name_p = "WHILE";

jerry-core/parser/js/js-scanner.h

@@ -32,6 +32,8 @@
 typedef enum
 {
   SCANNER_TYPE_END, /**< mark the last info block */
+  SCANNER_TYPE_END_ARGUMENTS, /**< mark the end of function arguments
+                               *   (only present if a function script is parsed) */
   SCANNER_TYPE_WHILE, /**< while statement */
   SCANNER_TYPE_FOR, /**< for statement */
   SCANNER_TYPE_FOR_IN, /**< for-in statement */

tests/jerry/es2015/function-param-init.js

@@ -79,3 +79,12 @@ CheckSyntaxError('function x(a =, b) {}');
 CheckSyntaxError('function x(a = (b) {}');
 CheckSyntaxError('function x(a, a = 5) {}');
 CheckSyntaxError('function x(a = 5, a) {}');
+
+// Pre-scanner tests.
+var str = "a = 5, b, c = function() { for (var a = 0; a < 4; a++) ; return a; } ()"
+
+var f = new Function (str, str);
+f();
+
+var f = new Function (str, "return (a + c) * (b == undefined ? 1 : 0)");
+assert (f() == 9);