Parser optimizations.

- parser is now non-recursive (i.e. parse function is not called recursively in any case);
 - byte-code is now more compact:
    - constants are now not immediately dumped upon occurence, but later - where necessary;
    - assignments are combined with unary / binary operations;
    - binary operations are encoded more compactly in many cases;
 - byte-code arrays are now allocated separately for each scope (so, GC of the scopes now becomes possible);
 - byte-code is dumped directly into corresponding byte-code arrays:
   - linked lists of op_meta are not now used for main code of a scope.

JerryScript-DCO-1.0-Signed-off-by: Ruben Ayrapetyan r.ayrapetyan@samsung.com
JerryScript-DCO-1.0-Signed-off-by: Andrey Shitov a.shitov@samsung.com

jerry-core/parser/js/lexer.h

@@ -25,7 +25,6 @@
 typedef enum __attr_packed___
 {
   /* Not a keyword.  */
-  KW_NONE = 0,
   KW_BREAK,
   KW_CASE,
   KW_CATCH,
@@ -82,15 +81,15 @@ typedef enum __attr_packed___
 /* Type of tokens.  */
 typedef enum __attr_packed___
 {
-  TOK_EOF = 0, // End of file
+  TOKEN_TYPE__BEGIN = 1,
+
+  TOK_EOF = TOKEN_TYPE__BEGIN, // End of file
   TOK_NAME, // Identifier
-  TOK_KEYWORD, // Keyword
   TOK_SMALL_INT,
   TOK_NUMBER,
 
   TOK_NULL,
   TOK_BOOL,
-  TOK_NEWLINE,
   TOK_STRING,
   TOK_OPEN_BRACE, // {
 
@@ -103,85 +102,178 @@ typedef enum __attr_packed___
   TOK_DOT, // .
   TOK_SEMICOLON, // ;
   TOK_COMMA, // ,
-  TOK_LESS, // <
-  TOK_GREATER, // >
 
-  TOK_LESS_EQ, // <=
-  TOK_GREATER_EQ, // <=
-  TOK_DOUBLE_EQ, // ==
-  TOK_NOT_EQ, // !=
-  TOK_TRIPLE_EQ, // ===
+  TOKEN_TYPE__UNARY_BEGIN,
+  TOKEN_TYPE__ADDITIVE_BEGIN = TOKEN_TYPE__UNARY_BEGIN,
 
-  TOK_NOT_DOUBLE_EQ, // !==
-  TOK_PLUS, // +
+  TOK_PLUS = TOKEN_TYPE__ADDITIVE_BEGIN, // +
   TOK_MINUS, // -
-  TOK_MULT, // *
-  TOK_MOD, // %
+
+  TOKEN_TYPE__ADDITIVE_END = TOK_MINUS,
 
   TOK_DOUBLE_PLUS, // ++
   TOK_DOUBLE_MINUS, // --
-  TOK_LSHIFT, // <<
+  TOK_NOT, // !
+  TOK_COMPL, // ~
+
+  TOKEN_TYPE__UNARY_END = TOK_COMPL, /* keywords are not listed
+                                      * in the range */
+
+  TOKEN_TYPE__MULTIPLICATIVE_BEGIN,
+
+  TOK_MULT = TOKEN_TYPE__MULTIPLICATIVE_BEGIN, // *
+  TOK_MOD, // %
+  TOK_DIV, // /
+
+  TOKEN_TYPE__MULTIPLICATIVE_END = TOK_DIV,
+
+  TOKEN_TYPE__SHIFT_BEGIN,
+
+  TOK_LSHIFT = TOKEN_TYPE__SHIFT_BEGIN, // <<
   TOK_RSHIFT, // >>
   TOK_RSHIFT_EX, // >>>
 
+  TOKEN_TYPE__SHIFT_END = TOK_RSHIFT_EX,
+
+  TOKEN_TYPE__RELATIONAL_BEGIN,
+
+  TOK_LESS = TOKEN_TYPE__RELATIONAL_BEGIN, // <
+  TOK_GREATER, // >
+  TOK_LESS_EQ, // <=
+  TOK_GREATER_EQ, // <=
+
+  TOKEN_TYPE__RELATIONAL_END = TOK_GREATER_EQ,
+
+  TOKEN_TYPE__EQUALITY_BEGIN,
+
+  TOK_DOUBLE_EQ = TOKEN_TYPE__EQUALITY_BEGIN, // ==
+  TOK_NOT_EQ, // !=
+  TOK_TRIPLE_EQ, // ===
+  TOK_NOT_DOUBLE_EQ, // !==
+
+  TOKEN_TYPE__EQUALITY_END = TOK_NOT_DOUBLE_EQ,
+
   TOK_AND, // &
   TOK_OR, // |
   TOK_XOR, // ^
-  TOK_NOT, // !
-  TOK_COMPL, // ~
 
   TOK_DOUBLE_AND, // &&
   TOK_DOUBLE_OR, // ||
   TOK_QUERY, // ?
   TOK_COLON, // :
-  TOK_EQ, // =
 
+  TOKEN_TYPE__ASSIGNMENTS_BEGIN,
+
+  TOK_EQ = TOKEN_TYPE__ASSIGNMENTS_BEGIN, // =
   TOK_PLUS_EQ, // +=
   TOK_MINUS_EQ, // -=
   TOK_MULT_EQ, // *=
   TOK_MOD_EQ, // %=
   TOK_LSHIFT_EQ, // <<=
-
   TOK_RSHIFT_EQ, // >>=
   TOK_RSHIFT_EX_EQ, // >>>=
   TOK_AND_EQ, // &=
   TOK_OR_EQ, // |=
   TOK_XOR_EQ, // ^=
-
-  TOK_DIV, // /
   TOK_DIV_EQ, // /=
+
+  TOKEN_TYPE__ASSIGNMENTS_END = TOK_DIV_EQ,
+
   TOK_EMPTY,
   TOK_REGEXP, // RegularExpressionLiteral (/.../gim)
-} token_type;
+
+  TOKEN_TYPE__KEYWORD_BEGIN,
+
+  TOK_KW_BREAK,
+  TOK_KW_CASE,
+  TOK_KW_CATCH,
+  TOK_KW_CLASS,
+
+  TOK_KW_CONST,
+  TOK_KW_CONTINUE,
+  TOK_KW_DEBUGGER,
+  TOK_KW_DEFAULT,
+  TOK_KW_DELETE,
+
+  TOK_KW_DO,
+  TOK_KW_ELSE,
+  TOK_KW_ENUM,
+  TOK_KW_EXPORT,
+  TOK_KW_EXTENDS,
+
+  TOK_KW_FINALLY,
+  TOK_KW_FOR,
+  TOK_KW_FUNCTION,
+  TOK_KW_IF,
+  TOK_KW_IN,
+
+  TOK_KW_INSTANCEOF,
+  TOK_KW_INTERFACE,
+  TOK_KW_IMPORT,
+  TOK_KW_IMPLEMENTS,
+  TOK_KW_LET,
+
+  TOK_KW_NEW,
+  TOK_KW_PACKAGE,
+  TOK_KW_PRIVATE,
+  TOK_KW_PROTECTED,
+  TOK_KW_PUBLIC,
+
+  TOK_KW_RETURN,
+  TOK_KW_STATIC,
+  TOK_KW_SUPER,
+  TOK_KW_SWITCH,
+  TOK_KW_THIS,
+
+  TOK_KW_THROW,
+  TOK_KW_TRY,
+  TOK_KW_TYPEOF,
+  TOK_KW_VAR,
+  TOK_KW_VOID,
+
+  TOK_KW_WHILE,
+  TOK_KW_WITH,
+  TOK_KW_YIELD,
+
+  TOKEN_TYPE__KEYWORD_END = TOK_KW_YIELD,
+
+  TOKEN_TYPE__END = TOKEN_TYPE__KEYWORD_END
+} jsp_token_type_t;
+
+/* Flags, describing token properties */
+typedef enum
+{
+  JSP_TOKEN_FLAG__NO_FLAGS            = 0x00, /* default flag */
+  JSP_TOKEN_FLAG_PRECEDED_BY_NEWLINES = 0x01  /* designates that newline precedes the token */
+} jsp_token_flag_t;
 
 typedef lit_utf8_iterator_pos_t locus;
 
 /* Represents the contents of a token.  */
 typedef struct
 {
-  locus loc;
-  token_type type;
-  uint16_t uid;
+  locus loc; /**< token location */
+  uint16_t uid; /**< encodes token's value, depending on token type */
+  uint8_t type; /**< token type */
+  uint8_t flags; /**< token flags */
 } token;
 
 /**
  * Initializer for empty token
  */
-#define TOKEN_EMPTY_INITIALIZER {LIT_ITERATOR_POS_ZERO, TOK_EMPTY, 0}
+#define TOKEN_EMPTY_INITIALIZER {LIT_ITERATOR_POS_ZERO, 0, TOK_EMPTY, JSP_TOKEN_FLAG__NO_FLAGS}
 
 void lexer_init (const jerry_api_char_t *, size_t, bool);
 
-token lexer_next_token (bool);
-void lexer_save_token (token);
-token lexer_prev_token (void);
+token lexer_next_token (bool, bool);
 
 void lexer_seek (locus);
 void lexer_locus_to_line_and_column (locus, size_t *, size_t *);
 void lexer_dump_line (size_t);
-const char *lexer_keyword_to_string (keyword);
-const char *lexer_token_type_to_string (token_type);
+const char *lexer_token_type_to_string (jsp_token_type_t);
 
-void lexer_set_strict_mode (bool);
+jsp_token_type_t lexer_get_token_type (token);
+bool lexer_is_preceded_by_newlines (token);
 
 extern bool lexer_are_tokens_with_same_identifier (token, token);