Add --test262-esnext option to run-tests.py (#4027)

Changes:
- Imported and unified test262 test harness for ES2015 and ESNext
- Simplified runner scripts accordingly
- Run tests on CI to be able detect regressions and progressions too

JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác csaba.osztrogonac@h-lab.eu

.github/workflows/gh-actions.yml

@@ -74,6 +74,13 @@ jobs:
       - name: Test262 - ES2015
         run: $RUNNER --test262-es2015 update
 
+  Conformance_Tests_ESNext:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Test262 - ESNext
+        run: $RUNNER --test262-esnext update
+
   Unit_Tests:
     runs-on: ubuntu-18.04
     steps:

tests/test262-es6.patch

@@ -1,24 +0,0 @@
-diff --git a/tools/packaging/test262.py b/tools/packaging/test262.py
-index 921360a05e..a19c8a14e6 100755
---- a/tools/packaging/test262.py
-+++ b/tools/packaging/test262.py
-@@ -168,6 +168,8 @@ class TestResult(object):
-     if len(err) > 0:
-        target.write("--- errors ---  \n %s" % err)
- 
-+    target.write("\n--- exit code: %d ---\n" % self.exit_code)
-+
-   # This is a way to make the output from the "whitespace" tests into valid XML
-   def SafeFormat(self, msg):
-     try:
-@@ -469,8 +471,8 @@ class TestSuite(object):
-           if self.ShouldRun(rel_path, tests):
-             basename = path.basename(full_path)[:-3]
-             name = rel_path.split(path.sep)[:-1] + [basename]
--            if EXCLUDE_LIST.count(basename) >= 1:
--              print 'Excluded: ' + basename
-+            if rel_path in EXCLUDE_LIST:
-+              print 'Excluded: ' + rel_path
-             else:
-               if not self.non_strict_only:
-                 strict_case = TestCase(self, name, full_path, True)

tools/pylint/pylintrc

@@ -303,7 +303,7 @@ ignore-imports=no
 [DESIGN]
 
 # Maximum number of arguments for function / method
-max-args=6
+max-args=10
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore
@@ -316,7 +316,7 @@ max-locals=20
 max-returns=6
 
 # Maximum number of branch for function / method body
-max-branches=15
+max-branches=20
 
 # Maximum number of statements in function / method body
 max-statements=75
@@ -325,7 +325,7 @@ max-statements=75
 max-parents=7
 
 # Maximum number of attributes for a class (see R0902).
-max-attributes=7
+max-attributes=10
 
 # Minimum number of public methods for a class (see R0903).
 min-public-methods=0

tools/run-tests.py

@@ -106,6 +106,11 @@ TEST262_ES2015_TEST_SUITE_OPTIONS = [
     Options('test262_tests_es2015', OPTIONS_PROFILE_ESNEXT + ['--line-info=on', '--error-messages=on']),
 ]
 
+# Test options for test262-esnext
+TEST262_ESNEXT_TEST_SUITE_OPTIONS = [
+    Options('test262_tests_esnext', OPTIONS_PROFILE_ESNEXT + ['--line-info=on', '--error-messages=on']),
+]
+
 # Test options for jerry-debugger
 DEBUGGER_TEST_OPTIONS = [
     Options('jerry_debugger_tests',
@@ -200,6 +205,10 @@ def get_arguments():
                         nargs='?', choices=['default', 'all', 'update'],
                         help='Run test262 - ES2015. default: all tests except excludelist, ' +
                         'all: all tests, update: all tests and update excludelist')
+    parser.add_argument('--test262-esnext', default=False, const='default',
+                        nargs='?', choices=['default', 'all', 'update'],
+                        help='Run test262 - ESnext. default: all tests except excludelist, ' +
+                        'all: all tests, update: all tests and update excludelist')
     parser.add_argument('--unittests', action='store_true',
                         help='Run unittests (including doctests)')
     parser.add_argument('--buildoption-test', action='store_true',
@@ -401,6 +410,8 @@ def run_test262_test_suite(options):
         jobs.extend(TEST262_TEST_SUITE_OPTIONS)
     if options.test262_es2015:
         jobs.extend(TEST262_ES2015_TEST_SUITE_OPTIONS)
+    if options.test262_esnext:
+        jobs.extend(TEST262_ESNEXT_TEST_SUITE_OPTIONS)
 
     for job in jobs:
         ret_build, build_dir_path = create_binary(job, options)
@@ -414,9 +425,12 @@ def run_test262_test_suite(options):
             '--test-dir', settings.TEST262_TEST_SUITE_DIR
         ]
 
-        if '--profile=es.next' in job.build_args:
+        if job.name.endswith('es2015'):
             test_cmd.append('--es2015')
             test_cmd.append(options.test262_es2015)
+        elif job.name.endswith('esnext'):
+            test_cmd.append('--esnext')
+            test_cmd.append(options.test262_esnext)
         else:
             test_cmd.append('--es51')
 
@@ -483,7 +497,7 @@ def main(options):
         Check(options.check_magic_strings, run_check, [settings.MAGIC_STRINGS_SCRIPT]),
         Check(options.jerry_debugger, run_jerry_debugger_tests, options),
         Check(options.jerry_tests, run_jerry_tests, options),
-        Check(options.test262 or options.test262_es2015, run_test262_test_suite, options),
+        Check(options.test262 or options.test262_es2015 or options.test262_esnext, run_test262_test_suite, options),
         Check(options.unittests, run_unittests, options),
         Check(options.buildoption_test, run_buildoption_test, options),
     ]

tools/runners/run-test-suite-test262.py

@@ -46,13 +46,29 @@ def get_arguments():
                        nargs='?', choices=['default', 'all', 'update'],
                        help='Run test262 - ES2015. default: all tests except excludelist, ' +
                        'all: all tests, update: all tests and update excludelist')
+    group.add_argument('--esnext', default=False, const='default',
+                       nargs='?', choices=['default', 'all', 'update'],
+                       help='Run test262 - ES.next. default: all tests except excludelist, ' +
+                       'all: all tests, update: all tests and update excludelist')
 
     args = parser.parse_args()
 
     if args.es2015:
         args.test_dir = os.path.join(args.test_dir, 'es2015')
+        args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__))
+        args.test262_git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5'
+        args.excludelist_path = os.path.join('tests', 'test262-es6-excludelist.xml')
+    elif args.esnext:
+        args.test_dir = os.path.join(args.test_dir, 'esnext')
+        args.test262_harness_dir = os.path.abspath(os.path.dirname(__file__))
+        args.test262_git_hash = '281eb10b2844929a7c0ac04527f5b42ce56509fd'
+        args.excludelist_path = os.path.join('tests', 'test262-esnext-excludelist.xml')
     else:
         args.test_dir = os.path.join(args.test_dir, 'es51')
+        args.test262_harness_dir = args.test_dir
+        args.test262_git_hash = 'es5-tests'
+
+    args.mode = args.es2015 or args.esnext
 
     return args
 
@@ -67,23 +83,10 @@ def prepare_test262_test_suite(args):
         print('Cloning test262 repository failed.')
         return return_code
 
-    if args.es2015:
+    return_code = subprocess.call(['git', 'checkout', args.test262_git_hash], cwd=args.test_dir)
-        git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5'
+    assert not return_code, 'Cloning test262 repository failed - invalid git revision.'
-    else:
-        git_hash = 'es5-tests'
 
-    return_code = subprocess.call(['git', 'checkout', git_hash], cwd=args.test_dir)
+    if args.es51:
-    if return_code:
-        print('Cloning test262 repository failed - invalid git revision.')
-        return return_code
-
-    if args.es2015:
-        return_code = subprocess.call(['git', 'apply', os.path.join('..', '..', 'test262-es6.patch')],
-                                      cwd=args.test_dir)
-        if return_code:
-            print('Applying test262-es6.patch failed')
-            return return_code
-    else:
         path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice')
         if os.path.isdir(path_to_remove):
             shutil.rmtree(path_to_remove)
@@ -95,22 +98,7 @@ def prepare_test262_test_suite(args):
     return 0
 
 
-def prepare_exclude_list(args):
-    if args.es2015 == 'all' or args.es2015 == 'update':
-        return_code = subprocess.call(['git', 'checkout', 'excludelist.xml'], cwd=args.test_dir)
-        if return_code:
-            print('Reverting excludelist.xml failed')
-            return return_code
-    elif args.es2015 == 'default':
-        shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
-                        os.path.join(args.test_dir, 'excludelist.xml'))
-
-    return 0
-
-
 def update_exclude_list(args):
-    assert args.es2015 == 'update', "Only --es2015 option supports updating excludelist"
-
     print("=== Summary - updating excludelist ===\n")
     failing_tests = set()
     new_passing_tests = set()
@@ -124,7 +112,7 @@ def update_exclude_list(args):
             elif line.startswith('Failed Tests'):
                 summary_found = True
 
-    with open(os.path.join('tests', 'test262-es6-excludelist.xml'), 'r+') as exclude_file:
+    with open(args.excludelist_path, 'r+') as exclude_file:
         lines = exclude_file.readlines()
         exclude_file.seek(0)
         exclude_file.truncate()
@@ -170,17 +158,13 @@ def main(args):
     if return_code:
         return return_code
 
-    return_code = prepare_exclude_list(args)
-    if return_code:
-        return return_code
-
     if sys.platform == 'win32':
         original_timezone = util.get_timezone()
         util.set_sighdl_to_reset_timezone(original_timezone)
         util.set_timezone('Pacific Standard Time')
 
     command = (args.runtime + ' ' + args.engine).strip()
-    if args.es2015:
+    if args.es2015 or args.esnext:
         try:
             subprocess.check_output(["timeout", "--version"])
             command = "timeout 3 " + command
@@ -190,11 +174,22 @@ def main(args):
     kwargs = {}
     if sys.version_info.major >= 3:
         kwargs['errors'] = 'ignore'
-    proc = subprocess.Popen(get_platform_cmd_prefix() +
+
-                            [os.path.join(args.test_dir, 'tools/packaging/test262.py'),
+    if args.es51:
+        test262_harness_path = os.path.join(args.test262_harness_dir, 'tools/packaging/test262.py')
+    else:
+        test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py')
+
+    test262_command = get_platform_cmd_prefix() + \
+                      [test262_harness_path,
                        '--command', command,
                        '--tests', args.test_dir,
-                             '--full-summary'],
+                       '--full-summary']
+
+    if 'excludelist_path' in args and args.mode == 'default':
+        test262_command.extend(['--exclude-list', args.excludelist_path])
+
+    proc = subprocess.Popen(test262_command,
                             universal_newlines=True,
                             stdout=subprocess.PIPE,
                             **kwargs)
@@ -226,7 +221,7 @@ def main(args):
     if sys.platform == 'win32':
         util.set_timezone(original_timezone)
 
-    if args.es2015 == 'update':
+    if args.mode == 'update':
         return_code = update_exclude_list(args)
 
     return return_code

tools/runners/test262-harness.py

@@ -0,0 +1,901 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This file is based on work under the following copyright and permission notice:
+# https://github.com/test262-utils/test262-harness-py
+# test262.py, _monkeyYaml.py, parseTestRecord.py
+
+# license of test262.py:
+# Copyright 2009 the Sputnik authors.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+# This is derived from sputnik.py, the Sputnik console test runner,
+# with elements from packager.py, which is separately
+# copyrighted. TODO: Refactor so there is less duplication between
+# test262.py and packager.py.
+
+# license of _packager.py:
+# Copyright (c) 2012 Ecma International.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+# license of _monkeyYaml.py:
+# Copyright 2014 by Sam Mikes.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+# license of parseTestRecord.py:
+# Copyright 2011 by Google, Inc.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+
+from __future__ import print_function
+
+import logging
+import optparse
+import os
+from os import path
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+import xml.dom.minidom
+from collections import Counter
+
+#######################################################################
+# based on _monkeyYaml.py
+#######################################################################
+
+M_YAML_LIST_PATTERN = re.compile(r"^\[(.*)\]$")
+M_YAML_MULTILINE_LIST = re.compile(r"^ *- (.*)$")
+
+
+def yaml_load(string):
+    return my_read_dict(string.splitlines())[1]
+
+
+def my_read_dict(lines, indent=""):
+    dictionary = {}
+    key = None
+    empty_lines = 0
+
+    while lines:
+        if not lines[0].startswith(indent):
+            break
+
+        line = lines.pop(0)
+        if my_is_all_spaces(line):
+            empty_lines += 1
+            continue
+
+        result = re.match(r"(.*?):(.*)", line)
+
+        if result:
+            if not dictionary:
+                dictionary = {}
+            key = result.group(1).strip()
+            value = result.group(2).strip()
+            (lines, value) = my_read_value(lines, value, indent)
+            dictionary[key] = value
+        else:
+            if dictionary and key and key in dictionary:
+                char = " " if empty_lines == 0 else "\n" * empty_lines
+                dictionary[key] += char + line.strip()
+            else:
+                raise Exception("monkeyYaml is confused at " + line)
+        empty_lines = 0
+
+    if not dictionary:
+        dictionary = None
+
+    return lines, dictionary
+
+
+def my_read_value(lines, value, indent):
+    if value == ">" or value == "|":
+        (lines, value) = my_multiline(lines, value == "|")
+        value = value + "\n"
+        return (lines, value)
+    if lines and not value:
+        if my_maybe_list(lines[0]):
+            return my_multiline_list(lines, value)
+        indent_match = re.match("(" + indent + r"\s+)", lines[0])
+        if indent_match:
+            if ":" in lines[0]:
+                return my_read_dict(lines, indent_match.group(1))
+            return my_multiline(lines, False)
+    return lines, my_read_one_line(value)
+
+
+def my_maybe_list(value):
+    return M_YAML_MULTILINE_LIST.match(value)
+
+
+def my_multiline_list(lines, value):
+    # assume no explcit indentor (otherwise have to parse value)
+    value = []
+    indent = None
+    while lines:
+        line = lines.pop(0)
+        leading = my_leading_spaces(line)
+        if my_is_all_spaces(line):
+            pass
+        elif leading < indent:
+            lines.insert(0, line)
+            break
+        else:
+            indent = indent or leading
+            value += [my_read_one_line(my_remove_list_header(indent, line))]
+    return (lines, value)
+
+
+def my_remove_list_header(indent, line):
+    line = line[indent:]
+    return M_YAML_MULTILINE_LIST.match(line).group(1)
+
+
+def my_read_one_line(value):
+    if M_YAML_LIST_PATTERN.match(value):
+        return my_flow_list(value)
+    elif re.match(r"^[-0-9]*$", value):
+        try:
+            value = int(value)
+        except ValueError:
+            pass
+    elif re.match(r"^[-.0-9eE]*$", value):
+        try:
+            value = float(value)
+        except ValueError:
+            pass
+    elif re.match(r"^('|\").*\1$", value):
+        value = value[1:-1]
+    return value
+
+
+def my_flow_list(value):
+    result = M_YAML_LIST_PATTERN.match(value)
+    values = result.group(1).split(",")
+    return [my_read_one_line(v.strip()) for v in values]
+
+
+def my_multiline(lines, preserve_newlines=False):
+    # assume no explcit indentor (otherwise have to parse value)
+    value = ""
+    indent = my_leading_spaces(lines[0])
+    was_empty = None
+
+    while lines:
+        line = lines.pop(0)
+        is_empty = my_is_all_spaces(line)
+
+        if is_empty:
+            if preserve_newlines:
+                value += "\n"
+        elif my_leading_spaces(line) < indent:
+            lines.insert(0, line)
+            break
+        else:
+            if preserve_newlines:
+                if was_empty != None:
+                    value += "\n"
+            else:
+                if was_empty:
+                    value += "\n"
+                elif was_empty is False:
+                    value += " "
+            value += line[(indent):]
+
+        was_empty = is_empty
+
+    return (lines, value)
+
+
+def my_is_all_spaces(line):
+    return len(line.strip()) == 0
+
+
+def my_leading_spaces(line):
+    return len(line) - len(line.lstrip(' '))
+
+
+#######################################################################
+# based on parseTestRecord.py
+#######################################################################
+
+# Matches trailing whitespace and any following blank lines.
+_BLANK_LINES = r"([ \t]*[\r\n]{1,2})*"
+
+# Matches the YAML frontmatter block.
+# It must be non-greedy because test262-es2015/built-ins/Object/assign/Override.js contains a comment like yaml pattern
+_YAML_PATTERN = re.compile(r"/\*---(.*?)---\*/" + _BLANK_LINES, re.DOTALL)
+
+# Matches all known variants for the license block.
+# https://github.com/tc39/test262/blob/705d78299cf786c84fa4df473eff98374de7135a/tools/lint/lib/checks/license.py
+_LICENSE_PATTERN = re.compile(
+    r'// Copyright( \([C]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
+    r'(' +
+    r'// This code is governed by the( BSD)? license found in the LICENSE file\.' +
+    r'|' +
+    r'// See LICENSE for details.' +
+    r'|' +
+    r'// Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
+    r'// found in the LICENSE file\.' +
+    r'|' +
+    r'// See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
+    r')' + _BLANK_LINES, re.IGNORECASE)
+
+
+def yaml_attr_parser(test_record, attrs, name, onerror=print):
+    parsed = yaml_load(attrs)
+    if parsed is None:
+        onerror("Failed to parse yaml in name %s" % name)
+        return
+
+    for key in parsed:
+        value = parsed[key]
+        if key == "info":
+            key = "commentary"
+        test_record[key] = value
+
+    if 'flags' in test_record:
+        for flag in test_record['flags']:
+            test_record[flag] = ""
+
+
+def find_license(src):
+    match = _LICENSE_PATTERN.search(src)
+    if not match:
+        return None
+
+    return match.group(0)
+
+
+def find_attrs(src):
+    match = _YAML_PATTERN.search(src)
+    if not match:
+        return (None, None)
+
+    return (match.group(0), match.group(1).strip())
+
+
+def parse_test_record(src, name, onerror=print):
+    # Find the license block.
+    header = find_license(src)
+
+    # Find the YAML frontmatter.
+    (frontmatter, attrs) = find_attrs(src)
+
+    # YAML frontmatter is required for all tests.
+    if frontmatter is None:
+        onerror("Missing frontmatter: %s" % name)
+
+    # The license shuold be placed before the frontmatter and there shouldn't be
+    # any extra content between the license and the frontmatter.
+    if header is not None and frontmatter is not None:
+        header_idx = src.index(header)
+        frontmatter_idx = src.index(frontmatter)
+        if header_idx > frontmatter_idx:
+            onerror("Unexpected license after frontmatter: %s" % name)
+
+        # Search for any extra test content, but ignore whitespace only or comment lines.
+        extra = src[header_idx + len(header): frontmatter_idx]
+        if extra and any(line.strip() and not line.lstrip().startswith("//") for line in extra.split("\n")):
+            onerror(
+                "Unexpected test content between license and frontmatter: %s" % name)
+
+    # Remove the license and YAML parts from the actual test content.
+    test = src
+    if frontmatter is not None:
+        test = test.replace(frontmatter, '')
+    if header is not None:
+        test = test.replace(header, '')
+
+    test_record = {}
+    test_record['header'] = header.strip() if header else ''
+    test_record['test'] = test
+
+    if attrs:
+        yaml_attr_parser(test_record, attrs, name, onerror)
+
+    # Report if the license block is missing in non-generated tests.
+    if header is None and "generated" not in test_record and "hashbang" not in name:
+        onerror("No license found in: %s" % name)
+
+    return test_record
+
+
+#######################################################################
+# based on test262.py
+#######################################################################
+
+class Test262Error(Exception):
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+
+def report_error(error_string):
+    raise Test262Error(error_string)
+
+
+def build_options():
+    result = optparse.OptionParser()
+    result.add_option("--command", default=None,
+                      help="The command-line to run")
+    result.add_option("--tests", default=path.abspath('.'),
+                      help="Path to the tests")
+    result.add_option("--exclude-list", default=None,
+                      help="Path to the excludelist.xml file")
+    result.add_option("--cat", default=False, action="store_true",
+                      help="Print packaged test code that would be run")
+    result.add_option("--summary", default=False, action="store_true",
+                      help="Print summary after running tests")
+    result.add_option("--full-summary", default=False, action="store_true",
+                      help="Print summary and test output after running tests")
+    result.add_option("--strict_only", default=False, action="store_true",
+                      help="Test only strict mode")
+    result.add_option("--non_strict_only", default=False, action="store_true",
+                      help="Test only non-strict mode")
+    result.add_option("--unmarked_default", default="both",
+                      help="default mode for tests of unspecified strictness")
+    result.add_option("--logname", help="Filename to save stdout to")
+    result.add_option("--loglevel", default="warning",
+                      help="sets log level to debug, info, warning, error, or critical")
+    result.add_option("--print-handle", default="print",
+                      help="Command to print from console")
+    result.add_option("--list-includes", default=False, action="store_true",
+                      help="List includes required by tests")
+    return result
+
+
+def validate_options(options):
+    if not options.command:
+        report_error("A --command must be specified.")
+    if not path.exists(options.tests):
+        report_error("Couldn't find test path '%s'" % options.tests)
+
+
+def is_windows():
+    actual_platform = platform.system()
+    return (actual_platform == 'Windows') or (actual_platform == 'Microsoft')
+
+
+class TempFile(object):
+
+    def __init__(self, suffix="", prefix="tmp", text=False):
+        self.suffix = suffix
+        self.prefix = prefix
+        self.text = text
+        self.file_desc = None
+        self.name = None
+        self.is_closed = False
+        self.open_file()
+
+    def open_file(self):
+        (self.file_desc, self.name) = tempfile.mkstemp(
+            suffix=self.suffix,
+            prefix=self.prefix,
+            text=self.text)
+
+    def write(self, string):
+        os.write(self.file_desc, string)
+
+    def read(self):
+        file_desc = file(self.name)
+        result = file_desc.read()
+        file_desc.close()
+        return result
+
+    def close(self):
+        if not self.is_closed:
+            self.is_closed = True
+            os.close(self.file_desc)
+
+    def dispose(self):
+        try:
+            self.close()
+            os.unlink(self.name)
+        except OSError as exception:
+            logging.error("Error disposing temp file: %s", str(exception))
+
+
+class TestResult(object):
+
+    def __init__(self, exit_code, stdout, stderr, case):
+        self.exit_code = exit_code
+        self.stdout = stdout
+        self.stderr = stderr
+        self.case = case
+
+    def report_outcome(self, long_format):
+        name = self.case.get_name()
+        mode = self.case.get_mode()
+        if self.has_unexpected_outcome():
+            if self.case.is_negative():
+                print("=== %s was expected to fail in %s, but didn't ===" % (name, mode))
+                print("--- expected error: %s ---\n" % self.case.get_negative_type())
+            else:
+                if long_format:
+                    print("=== %s failed in %s ===" % (name, mode))
+                else:
+                    print("%s in %s: " % (name, mode))
+            self.write_output(sys.stdout)
+            if long_format:
+                print("===")
+        elif self.case.is_negative():
+            print("%s failed in %s as expected" % (name, mode))
+        else:
+            print("%s passed in %s" % (name, mode))
+
+    def write_output(self, target):
+        out = self.stdout.strip()
+        if out:
+            target.write("--- output --- \n %s" % out)
+        error = self.stderr.strip()
+        if error:
+            target.write("--- errors ---  \n %s" % error)
+
+        target.write("\n--- exit code: %d ---\n" % self.exit_code)
+
+    def has_failed(self):
+        return self.exit_code != 0
+
+    def async_has_failed(self):
+        return 'Test262:AsyncTestComplete' not in self.stdout
+
+    def has_unexpected_outcome(self):
+        if self.case.is_async_test():
+            return self.async_has_failed() or self.has_failed()
+        elif self.case.is_negative():
+            return not (self.has_failed() and self.case.negative_match(self.get_error_output()))
+
+        return self.has_failed()
+
+    def get_error_output(self):
+        if self.stderr:
+            return self.stderr
+        return self.stdout
+
+
+class TestCase(object):
+
+    def __init__(self, suite, name, full_path, strict_mode):
+        self.suite = suite
+        self.name = name
+        self.full_path = full_path
+        self.strict_mode = strict_mode
+        with open(self.full_path) as file_desc:
+            self.contents = file_desc.read()
+        test_record = parse_test_record(self.contents, name)
+        self.test = test_record["test"]
+        del test_record["test"]
+        del test_record["header"]
+        test_record.pop("commentary", None)    # do not throw if missing
+        self.test_record = test_record
+
+        self.validate()
+
+    def negative_match(self, stderr):
+        neg = re.compile(self.get_negative_type())
+        return re.search(neg, stderr)
+
+    def get_negative(self):
+        if not self.is_negative():
+            return None
+        return self.test_record["negative"]
+
+    def get_negative_type(self):
+        negative = self.get_negative()
+        if isinstance(negative, dict) and "type" in negative:
+            return negative["type"]
+        return negative
+
+    def get_negative_phase(self):
+        negative = self.get_negative()
+        return negative and "phase" in negative and negative["phase"]
+
+    def get_name(self):
+        return path.join(*self.name)
+
+    def get_mode(self):
+        if self.strict_mode:
+            return "strict mode"
+        return "non-strict mode"
+
+    def get_path(self):
+        return self.name
+
+    def is_negative(self):
+        return 'negative' in self.test_record
+
+    def is_only_strict(self):
+        return 'onlyStrict' in self.test_record
+
+    def is_no_strict(self):
+        return 'noStrict' in self.test_record or self.is_raw()
+
+    def is_raw(self):
+        return 'raw' in self.test_record
+
+    def is_async_test(self):
+        return 'async' in self.test_record or '$DONE' in self.test
+
+    def get_include_list(self):
+        if self.test_record.get('includes'):
+            return self.test_record['includes']
+        return []
+
+    def get_additional_includes(self):
+        return '\n'.join([self.suite.get_include(include) for include in self.get_include_list()])
+
+    def get_source(self):
+        if self.is_raw():
+            return self.test
+
+        source = self.suite.get_include("sta.js") + \
+            self.suite.get_include("assert.js")
+
+        if self.is_async_test():
+            source = source + \
+                self.suite.get_include("timer.js") + \
+                self.suite.get_include("doneprintHandle.js").replace(
+                    'print', self.suite.print_handle)
+
+        source = source + \
+            self.get_additional_includes() + \
+            self.test + '\n'
+
+        if self.get_negative_phase() == "early":
+            source = ("throw 'Expected an early error, but code was executed.';\n" +
+                      source)
+
+        if self.strict_mode:
+            source = '"use strict";\nvar strict_mode = true;\n' + source
+        else:
+            # add comment line so line numbers match in both strict and non-strict version
+            source = '//"no strict";\nvar strict_mode = false;\n' + source
+
+        return source
+
+    @staticmethod
+    def instantiate_template(template, params):
+        def get_parameter(match):
+            key = match.group(1)
+            return params.get(key, match.group(0))
+
+        return re.sub(r"\{\{(\w+)\}\}", get_parameter, template)
+
+    @staticmethod
+    def execute(command):
+        if is_windows():
+            args = '%s' % command
+        else:
+            args = command.split(" ")
+        stdout = TempFile(prefix="test262-out-")
+        stderr = TempFile(prefix="test262-err-")
+        try:
+            logging.info("exec: %s", str(args))
+            process = subprocess.Popen(
+                args,
+                shell=is_windows(),
+                stdout=stdout.file_desc,
+                stderr=stderr.file_desc
+            )
+            code = process.wait()
+            out = stdout.read()
+            err = stderr.read()
+        finally:
+            stdout.dispose()
+            stderr.dispose()
+        return (code, out, err)
+
+    def run_test_in(self, command_template, tmp):
+        tmp.write(self.get_source())
+        tmp.close()
+        command = TestCase.instantiate_template(command_template, {
+            'path': tmp.name
+        })
+        (code, out, err) = TestCase.execute(command)
+        return TestResult(code, out, err, self)
+
+    def run(self, command_template):
+        tmp = TempFile(suffix=".js", prefix="test262-", text=True)
+        try:
+            result = self.run_test_in(command_template, tmp)
+        finally:
+            tmp.dispose()
+        return result
+
+    def print_source(self):
+        print(self.get_source())
+
+    def validate(self):
+        flags = self.test_record.get("flags")
+        phase = self.get_negative_phase()
+
+        if phase not in [None, False, "parse", "early", "runtime", "resolution"]:
+            raise TypeError("Invalid value for negative phase: " + phase)
+
+        if not flags:
+            return
+
+        if 'raw' in flags:
+            if 'noStrict' in flags:
+                raise TypeError("The `raw` flag implies the `noStrict` flag")
+            elif 'onlyStrict' in flags:
+                raise TypeError(
+                    "The `raw` flag is incompatible with the `onlyStrict` flag")
+            elif self.get_include_list():
+                raise TypeError(
+                    "The `raw` flag is incompatible with the `includes` tag")
+
+
+class ProgressIndicator(object):
+
+    def __init__(self, count):
+        self.count = count
+        self.succeeded = 0
+        self.failed = 0
+        self.failed_tests = []
+
+    def has_run(self, result):
+        result.report_outcome(True)
+        if result.has_unexpected_outcome():
+            self.failed += 1
+            self.failed_tests.append(result)
+        else:
+            self.succeeded += 1
+
+
+def make_plural(num):
+    if num == 1:
+        return (num, "")
+    return (num, "s")
+
+
+def percent_format(partial, total):
+    return "%i test%s (%.1f%%)" % (make_plural(partial) +
+                                   ((100.0 * partial)/total,))
+
+
+class TestSuite(object):
+
+    def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle, exclude_list_path):
+        self.test_root = path.join(root, 'test')
+        self.lib_root = path.join(root, 'harness')
+        self.strict_only = strict_only
+        self.non_strict_only = non_strict_only
+        self.unmarked_default = unmarked_default
+        self.print_handle = print_handle
+        self.include_cache = {}
+        self.exclude_list = []
+        self.logf = None
+
+        if exclude_list_path:
+            if os.path.exists(exclude_list_path):
+                self.exclude_list = xml.dom.minidom.parse(exclude_list_path)
+                self.exclude_list = self.exclude_list.getElementsByTagName("test")
+                self.exclude_list = [x.getAttribute("id") for x in self.exclude_list]
+            else:
+                report_error("Couldn't find excludelist '%s'" % exclude_list_path)
+
+    def validate(self):
+        if not path.exists(self.test_root):
+            report_error("No test repository found")
+        if not path.exists(self.lib_root):
+            report_error("No test library found")
+
+    @staticmethod
+    def is_hidden(test_path):
+        return test_path.startswith('.') or test_path == 'CVS'
+
+    @staticmethod
+    def is_test_case(test_path):
+        return test_path.endswith('.js') and not test_path.endswith('_FIXTURE.js')
+
+    @staticmethod
+    def should_run(rel_path, tests):
+        if not tests:
+            return True
+        for test in tests:
+            if test in rel_path:
+                return True
+        return False
+
+    def get_include(self, name):
+        if not name in self.include_cache:
+            static = path.join(self.lib_root, name)
+            if path.exists(static):
+                with open(static) as file_desc:
+                    contents = file_desc.read()
+                    contents = re.sub(r'\r\n', '\n', contents)
+                    self.include_cache[name] = contents + "\n"
+            else:
+                report_error("Can't find: " + static)
+        return self.include_cache[name]
+
+    def enumerate_tests(self, tests):
+        logging.info("Listing tests in %s", self.test_root)
+        cases = []
+        for root, dirs, files in os.walk(self.test_root):
+            for hidden_dir in [x for x in dirs if self.is_hidden(x)]:
+                dirs.remove(hidden_dir)
+            dirs.sort()
+            for test_path in filter(TestSuite.is_test_case, sorted(files)):
+                full_path = path.join(root, test_path)
+                if full_path.startswith(self.test_root):
+                    rel_path = full_path[len(self.test_root)+1:]
+                else:
+                    logging.warning("Unexpected path %s", full_path)
+                    rel_path = full_path
+                if self.should_run(rel_path, tests):
+                    basename = path.basename(full_path)[:-3]
+                    name = rel_path.split(path.sep)[:-1] + [basename]
+                    if rel_path in self.exclude_list:
+                        print('Excluded: ' + rel_path)
+                    else:
+                        if not self.non_strict_only:
+                            strict_case = TestCase(self, name, full_path, True)
+                            if not strict_case.is_no_strict():
+                                if strict_case.is_only_strict() or self.unmarked_default in ['both', 'strict']:
+                                    cases.append(strict_case)
+                        if not self.strict_only:
+                            non_strict_case = TestCase(self, name, full_path, False)
+                            if not non_strict_case.is_only_strict():
+                                if non_strict_case.is_no_strict() or self.unmarked_default in ['both', 'non_strict']:
+                                    cases.append(non_strict_case)
+        logging.info("Done listing tests")
+        return cases
+
+    def print_summary(self, progress, logfile):
+
+        def write(string):
+            if logfile:
+                self.logf.write(string + "\n")
+            print(string)
+
+        print("")
+        write("=== Summary ===")
+        count = progress.count
+        succeeded = progress.succeeded
+        failed = progress.failed
+        write(" - Ran %i test%s" % make_plural(count))
+        if progress.failed == 0:
+            write(" - All tests succeeded")
+        else:
+            write(" - Passed " + percent_format(succeeded, count))
+            write(" - Failed " + percent_format(failed, count))
+            positive = [c for c in progress.failed_tests if not c.case.is_negative()]
+            negative = [c for c in progress.failed_tests if c.case.is_negative()]
+            if positive:
+                print("")
+                write("Failed Tests")
+                for result in positive:
+                    write("  %s in %s" % (result.case.get_name(), result.case.get_mode()))
+            if negative:
+                print("")
+                write("Expected to fail but passed ---")
+                for result in negative:
+                    write("  %s in %s" % (result.case.get_name(), result.case.get_mode()))
+
+    def print_failure_output(self, progress, logfile):
+        for result in progress.failed_tests:
+            if logfile:
+                self.write_log(result)
+            print("")
+            result.report_outcome(False)
+
+    def run(self, command_template, tests, print_summary, full_summary, logname):
+        if not "{{path}}" in command_template:
+            command_template += " {{path}}"
+        cases = self.enumerate_tests(tests)
+        if not cases:
+            report_error("No tests to run")
+        progress = ProgressIndicator(len(cases))
+        if logname:
+            self.logf = open(logname, "w")
+
+        for case in cases:
+            result = case.run(command_template)
+            if logname:
+                self.write_log(result)
+            progress.has_run(result)
+
+        if print_summary:
+            self.print_summary(progress, logname)
+            if full_summary:
+                self.print_failure_output(progress, logname)
+            else:
+                print("")
+                print("Use --full-summary to see output from failed tests")
+        print("")
+        return progress.failed
+
+    def write_log(self, result):
+        name = result.case.get_name()
+        mode = result.case.get_mode()
+        if result.has_unexpected_outcome():
+            if result.case.is_negative():
+                self.logf.write(
+                    "=== %s was expected to fail in %s, but didn't === \n" % (name, mode))
+                self.logf.write("--- expected error: %s ---\n" % result.case.GetNegativeType())
+                result.write_output(self.logf)
+            else:
+                self.logf.write("=== %s failed in %s === \n" % (name, mode))
+                result.write_output(self.logf)
+            self.logf.write("===\n")
+        elif result.case.is_negative():
+            self.logf.write("%s failed in %s as expected \n" % (name, mode))
+        else:
+            self.logf.write("%s passed in %s \n" % (name, mode))
+
+    def print_source(self, tests):
+        cases = self.enumerate_tests(tests)
+        if cases:
+            cases[0].print_source()
+
+    def list_includes(self, tests):
+        cases = self.enumerate_tests(tests)
+        includes_dict = Counter()
+        for case in cases:
+            includes = case.get_include_list()
+            includes_dict.update(includes)
+
+        print(includes_dict)
+
+
+def main():
+    code = 0
+    parser = build_options()
+    (options, args) = parser.parse_args()
+    validate_options(options)
+
+    test_suite = TestSuite(options.tests,
+                           options.strict_only,
+                           options.non_strict_only,
+                           options.unmarked_default,
+                           options.print_handle,
+                           options.exclude_list)
+
+    test_suite.validate()
+    if options.loglevel == 'debug':
+        logging.basicConfig(level=logging.DEBUG)
+    elif options.loglevel == 'info':
+        logging.basicConfig(level=logging.INFO)
+    elif options.loglevel == 'warning':
+        logging.basicConfig(level=logging.WARNING)
+    elif options.loglevel == 'error':
+        logging.basicConfig(level=logging.ERROR)
+    elif options.loglevel == 'critical':
+        logging.basicConfig(level=logging.CRITICAL)
+
+    if options.cat:
+        test_suite.print_source(args)
+    elif options.list_includes:
+        test_suite.list_includes(args)
+    else:
+        code = test_suite.run(options.command, args,
+                              options.summary or options.full_summary,
+                              options.full_summary,
+                              options.logname)
+    return code
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except Test262Error as exception:
+        print("Error: %s" % exception.message)
+        sys.exit(1)