Optimize test262 runner (#4120)

Changes:
- Add new option to run-tests.py: --test262-test-list to run selected tests only
- Fix exclude list updater accordingly
- Run ESNext tests on GitHub CI in two batches to decrease runtime

JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác csaba.osztrogonac@h-lab.eu

.github/workflows/gh-actions.yml

@@ -79,16 +79,28 @@ jobs:
           name: Test262-ES2015-results
           path: build/tests/test262_tests_es2015/local/bin/test262.report
 
-  Conformance_Tests_ESNext:
+  Conformance_Tests_ESNext_A:
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v1
-      - name: Test262 - ESNext
+      - name: Test262 - ESNext (built-ins,annexB,harness,intl402)
-        run: $RUNNER --test262-esnext update
+        run: $RUNNER --test262-esnext update --test262-test-list built-ins,annexB,harness,intl402
       - uses: actions/upload-artifact@v2
         if: success() || failure()
         with:
-          name: Test262-ESNext-results
+          name: Test262-ESNext-results-A
+          path: build/tests/test262_tests_esnext/local/bin/test262.report
+
+  Conformance_Tests_ESNext_B:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Test262 - ESNext (language)
+        run: $RUNNER --test262-esnext update --test262-test-list language
+      - uses: actions/upload-artifact@v2
+        if: success() || failure()
+        with:
+          name: Test262-ESNext-results-B
           path: build/tests/test262_tests_esnext/local/bin/test262.report
 
   Unit_Tests:

tools/pylint/pylintrc

@@ -316,7 +316,7 @@ max-locals=20
 max-returns=6
 
 # Maximum number of branch for function / method body
-max-branches=20
+max-branches=25
 
 # Maximum number of statements in function / method body
 max-statements=75

tools/run-tests.py

@@ -209,6 +209,8 @@ def get_arguments():
                         nargs='?', choices=['default', 'all', 'update'],
                         help='Run test262 - ESnext. default: all tests except excludelist, ' +
                         'all: all tests, update: all tests and update excludelist')
+    parser.add_argument('--test262-test-list', metavar='LIST',
+                        help='Add a comma separated list of tests or directories to run in test262 test suite')
     parser.add_argument('--unittests', action='store_true',
                         help='Run unittests (including doctests)')
     parser.add_argument('--buildoption-test', action='store_true',
@@ -222,6 +224,12 @@ def get_arguments():
 
     script_args = parser.parse_args()
 
+    if script_args.test262_test_list and not \
+       (script_args.test262 or script_args.test262_es2015 or script_args.test262_esnext):
+        print("--test262-test-list is only allowed with --test262 or --test262-es2015 or --test262-esnext\n")
+        parser.print_help()
+        sys.exit(1)
+
     return script_args
 
 BINARY_CACHE = {}
@@ -437,6 +445,10 @@ def run_test262_test_suite(options):
         if job.test_args:
             test_cmd.extend(job.test_args)
 
+        if options.test262_test_list:
+            test_cmd.append('--test262-test-list')
+            test_cmd.append(options.test262_test_list)
+
         ret_test |= run_check(test_cmd, env=dict(TZ='America/Los_Angeles'))
 
     return ret_build | ret_test

tools/runners/run-test-suite-test262.py

@@ -50,6 +50,8 @@ def get_arguments():
                        nargs='?', choices=['default', 'all', 'update'],
                        help='Run test262 - ES.next. default: all tests except excludelist, ' +
                        'all: all tests, update: all tests and update excludelist')
+    parser.add_argument('--test262-test-list', metavar='LIST',
+                        help='Add a comma separated list of tests or directories to run in test262 test suite')
 
     args = parser.parse_args()
 
@@ -100,17 +102,21 @@ def prepare_test262_test_suite(args):
 
 def update_exclude_list(args):
     print("=== Summary - updating excludelist ===\n")
+    passing_tests = set()
     failing_tests = set()
     new_passing_tests = set()
     with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'r') as report_file:
-        summary_found = False
         for line in report_file:
-            if summary_found:
+            match = re.match('(=== )?(.*) (?:failed|passed) in (?:non-strict|strict)', line)
-                match = re.match(r"  (\S*) in ", line)
+            if match:
-                if match:
+                (unexpected, test) = match.groups()
-                    failing_tests.add(match.group(1) + '.js')
+                if unexpected:
-            elif line.startswith('Failed Tests'):
+                    failing_tests.add(test + '.js')
-                summary_found = True
+                else:
+                    passing_tests.add(test + '.js')
+
+    # Tests pass in strict-mode but fail in non-strict-mode (or vice versa) should be considered as failures
+    passing_tests = passing_tests - failing_tests
 
     with open(args.excludelist_path, 'r+') as exclude_file:
         lines = exclude_file.readlines()
@@ -125,8 +131,10 @@ def update_exclude_list(args):
                 if test in failing_tests:
                     failing_tests.remove(test)
                     exclude_file.write(line)
-                else:
+                elif test in passing_tests:
                     new_passing_tests.add(test)
+                else:
+                    exclude_file.write(line)
             else:
                 exclude_file.write(line)
 
@@ -189,6 +197,9 @@ def main(args):
     if 'excludelist_path' in args and args.mode == 'default':
         test262_command.extend(['--exclude-list', args.excludelist_path])
 
+    if args.test262_test_list:
+        test262_command.extend(args.test262_test_list.split(','))
+
     proc = subprocess.Popen(test262_command,
                             universal_newlines=True,
                             stdout=subprocess.PIPE,

tools/runners/test262-harness.py

@@ -424,7 +424,7 @@ class TestResult(object):
         mode = self.case.get_mode()
         if self.has_unexpected_outcome():
             if self.case.is_negative():
-                print("=== %s was expected to fail in %s, but didn't ===" % (name, mode))
+                print("=== %s passed in %s, but was expected to fail ===" % (name, mode))
                 print("--- expected error: %s ---\n" % self.case.get_negative_type())
             else:
                 if long_format:
@@ -829,7 +829,7 @@ class TestSuite(object):
         if result.has_unexpected_outcome():
             if result.case.is_negative():
                 self.logf.write(
-                    "=== %s was expected to fail in %s, but didn't === \n" % (name, mode))
+                    "=== %s passed in %s, but was expected to fail === \n" % (name, mode))
                 self.logf.write("--- expected error: %s ---\n" % result.case.GetNegativeType())
                 result.write_output(self.logf)
             else: