Improve test262 ES2015 runner (#3892)

Changes:
 - Add options to the test runner to be able to update exclude list
 - Update the exclude list
 - Make Travis CI signal if the exclude list isn't up-to-date

JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác oszi@inf.u-szeged.hu

.travis.yml

@@ -67,7 +67,7 @@ matrix:
 
     - name: "Conformance Tests - ES2015"
       env:
-        - OPTS="--test262-es2015"
+        - OPTS="--test262-es2015 update"
 
     - name: "Unit Tests"
       env:

tests/test262-es6-excludelist.xml

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <excludeList>
-  <test id="annexB/B.2.2.1.2.js"><reason></reason></test>
   <test id="annexB/B.2.3.10.js"><reason></reason></test>
   <test id="annexB/B.2.3.11.js"><reason></reason></test>
   <test id="annexB/B.2.3.12.js"><reason></reason></test>
@@ -416,7 +415,6 @@
   <test id="language/statements/class/definition/this-check-ordering.js"><reason></reason></test>
   <test id="language/statements/class/name-binding/in-extends-expression-assigned.js"><reason></reason></test>
   <test id="language/statements/class/name-binding/in-extends-expression.js"><reason></reason></test>
-  <test id="language/statements/class/strict-mode/arguments-caller.js"><reason></reason></test>
   <test id="language/statements/class/syntax/early-errors/class-body-static-method-get-propname-prototype.js"><reason></reason></test>
   <test id="language/statements/continue/labeled-continue.js"><reason></reason></test>
   <test id="language/statements/continue/nested-let-bound-for-loops-labeled-continue.js"><reason></reason></test>

tools/run-tests.py

@@ -194,8 +194,10 @@ def get_arguments():
                         help='Run jerry-tests')
     parser.add_argument('--test262', action='store_true',
                         help='Run test262 - ES5.1')
-    parser.add_argument('--test262-es2015', action='store_true',
+    parser.add_argument('--test262-es2015', default=False, const='default',
-                        help='Run test262 - ES2015')
+                        nargs='?', choices=['default', 'all', 'update'],
+                        help='Run test262 - ES2015. default: all tests except excludelist, ' +
+                        'all: all tests, update: all tests and update excludelist')
     parser.add_argument('--unittests', action='store_true',
                         help='Run unittests (including doctests)')
     parser.add_argument('--buildoption-test', action='store_true',
@@ -412,6 +414,7 @@ def run_test262_test_suite(options):
 
         if '--profile=es.next' in job.build_args:
             test_cmd.append('--es2015')
+            test_cmd.append(options.test262_es2015)
         else:
             test_cmd.append('--es51')
 

tools/runners/run-test-suite-test262.py

@@ -17,6 +17,7 @@
 from __future__ import print_function
 import argparse
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -41,8 +42,10 @@ def get_arguments():
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('--es51', action='store_true',
                        help='Run test262 ES5.1 version')
-    group.add_argument('--es2015', action='store_true',
+    group.add_argument('--es2015', default=False, const='default',
-                       help='Run test262 ES2015 version')
+                       nargs='?', choices=['default', 'all', 'update'],
+                       help='Run test262 - ES2015. default: all tests except excludelist, ' +
+                       'all: all tests, update: all tests and update excludelist')
 
     args = parser.parse_args()
 
@@ -75,15 +78,11 @@ def prepare_test262_test_suite(args):
         return return_code
 
     if args.es2015:
-        shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
-                        os.path.join(args.test_dir, 'excludelist.xml'))
-
         return_code = subprocess.call(['git', 'apply', os.path.join('..', '..', 'test262-es6.patch')],
                                       cwd=args.test_dir)
         if return_code:
             print('Applying test262-es6.patch failed')
             return return_code
-
     else:
         path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice')
         if os.path.isdir(path_to_remove):
@@ -95,23 +94,109 @@ def prepare_test262_test_suite(args):
 
     return 0
 
+
+def prepare_exclude_list(args):
+    if args.es2015 == 'all' or args.es2015 == 'update':
+        return_code = subprocess.call(['git', 'checkout', 'excludelist.xml'], cwd=args.test_dir)
+        if return_code:
+            print('Reverting excludelist.xml failed')
+            return return_code
+    elif args.es2015 == 'default':
+        shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
+                        os.path.join(args.test_dir, 'excludelist.xml'))
+
+    return 0
+
+
+def update_exclude_list(args):
+    assert args.es2015 == 'update', "Only --es2015 option supports updating excludelist"
+
+    print("=== Summary - updating excludelist ===\n")
+    failing_tests = set()
+    new_passing_tests = set()
+    with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'r') as report_file:
+        summary_found = False
+        for line in report_file:
+            if summary_found:
+                match = re.match(r"  (\S*) ", line)
+                if match:
+                    failing_tests.add(match.group(1) + '.js')
+            elif line.startswith('Failed Tests'):
+                summary_found = True
+
+    with open(os.path.join('tests', 'test262-es6-excludelist.xml'), 'r+') as exclude_file:
+        lines = exclude_file.readlines()
+        exclude_file.seek(0)
+        exclude_file.truncate()
+        exclude_file.write('<?xml version="1.0" encoding="utf-8" ?>\n')
+        exclude_file.write('<excludeList>\n')
+
+        for line in lines:
+            match = re.match(r"  <test id=\"(\S*)\">", line)
+            if match:
+                test = match.group(1)
+                if test in failing_tests:
+                    failing_tests.remove(test)
+                    exclude_file.write(line)
+                else:
+                    new_passing_tests.add(test)
+
+        if failing_tests:
+            print("New failing tests added to the excludelist")
+            for test in sorted(failing_tests):
+                exclude_file.write('  <test id="' + test + '"><reason></reason></test>\n')
+                print("  " + test)
+            print("")
+
+        exclude_file.write('</excludeList>\n')
+
+    if new_passing_tests:
+        print("New passing tests removed from the excludelist")
+        for test in sorted(new_passing_tests):
+            print("  " + test)
+        print("")
+
+    if failing_tests or new_passing_tests:
+        print("Excludelist was updated succesfully.")
+        return 1
+
+    print("Excludelist was already up-to-date.")
+    return 0
+
+
 def main(args):
     return_code = prepare_test262_test_suite(args)
     if return_code:
         return return_code
 
+    return_code = prepare_exclude_list(args)
+    if return_code:
+        return return_code
+
     if sys.platform == 'win32':
         original_timezone = util.get_timezone()
         util.set_sighdl_to_reset_timezone(original_timezone)
         util.set_timezone('Pacific Standard Time')
 
+    command = (args.runtime + ' ' + args.engine).strip()
+    if args.es2015:
+        try:
+            subprocess.check_output(["timeout", "--version"])
+            command = "timeout 3 " + command
+        except subprocess.CalledProcessError:
+            pass
+
+    kwargs = {}
+    if sys.version_info.major >= 3:
+        kwargs['errors'] = 'ignore'
     proc = subprocess.Popen(get_platform_cmd_prefix() +
                             [os.path.join(args.test_dir, 'tools/packaging/test262.py'),
-                             '--command', (args.runtime + ' ' + args.engine).strip(),
+                             '--command', command,
                              '--tests', args.test_dir,
                              '--summary'],
                             universal_newlines=True,
-                            stdout=subprocess.PIPE)
+                            stdout=subprocess.PIPE,
+                            **kwargs)
 
     return_code = 1
     with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'w') as output_file:
@@ -140,6 +225,9 @@ def main(args):
     if sys.platform == 'win32':
         util.set_timezone(original_timezone)
 
+    if args.es2015 == 'update':
+        return_code = update_exclude_list(args)
+
     return return_code