Allow to skip certain JS-tests under tests/jerry/ and tests/jerry-test-suite/ (#1632)

Add `--skip-list` option to tools/run-tests.py.

Additional modifications:
 - The test `parser-oom.js` doesn't work when system-allocator is enabled,
   hence we skip it in these jobs.
 - The sanitizer-tasks become mandatory.

JerryScript-DCO-1.0-Signed-off-by: Zsolt Borbély zsborbely.u-szeged@partner.samsung.com

.travis.yml

@@ -27,7 +27,7 @@ matrix:
           packages:
             - gcc-5
             - gcc-5-multilib
-      env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600
+      env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-O2,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600
     - compiler: gcc-5
       addons:
         apt:
@@ -36,9 +36,8 @@ matrix:
           packages:
             - gcc-5
             - gcc-5-multilib
-      env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600
+      env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600
   allow_failures:
-    - compiler: gcc-5
     - env: OPTS="--check-pylint"
   fast_finish: true
 

tools/run-tests.py

@@ -25,6 +25,7 @@ OUTPUT_DIR = path.join(PROJECT_DIR, 'build', 'tests')
 parser = argparse.ArgumentParser()
 parser.add_argument('--toolchain', action='store', default='', help='Add toolchain file')
 parser.add_argument('--buildoptions', action='store', default='', help='Add a comma separated list of extra build options to each test')
+parser.add_argument('--skip-list', action='store', default='', help='Add a comma separated list of patterns of the excluded JS-tests')
 parser.add_argument('--outdir', action='store', default=OUTPUT_DIR, help='Specify output directory (default: %(default)s)')
 parser.add_argument('--check-signed-off', action='store_true', default=False, help='Run signed-off check')
 parser.add_argument('--check-signed-off-tolerant', action='store_true', default=False, help='Run signed-off check in tolerant mode')
@@ -184,6 +185,9 @@ def run_jerry_tests():
             break
 
         test_cmd = [TEST_RUNNER_SCRIPT, get_binary_path(job.out_dir), JERRY_TESTS_DIR]
+        if script_args.skip_list:
+            test_cmd.append("--skip-list=" + script_args.skip_list)
+
         if job.test_args:
             test_cmd.extend(job.test_args)
 
@@ -207,6 +211,9 @@ def run_jerry_test_suite():
         else:
             test_cmd.append(JERRY_TEST_SUITE_ES51_LIST)
 
+        if script_args.skip_list:
+            test_cmd.append("--skip-list=" + script_args.skip_list)
+
         if job.test_args:
             test_cmd.extend(job.test_args)
 

tools/runners/run-test-suite.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Usage:
-#       ./tools/runners/run-test-suite.sh ENGINE TESTS [--snapshot] ENGINE_ARGS....
+#       ./tools/runners/run-test-suite.sh ENGINE TESTS [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS....
 
 TIMEOUT=${TIMEOUT:=5}
 
@@ -32,6 +32,13 @@ TEST_FILES=$OUTPUT_DIR/$TESTS_BASENAME.files
 TEST_FAILED=$OUTPUT_DIR/$TESTS_BASENAME.failed
 TEST_PASSED=$OUTPUT_DIR/$TESTS_BASENAME.passed
 
+if [[ "$1" =~ ^--skip-list=.* ]]
+then
+    SKIP_LIST=${1#--skip-list=}
+    SKIP_LIST=(${SKIP_LIST//,/ })
+    shift
+fi
+
 if [ "$1" == "--snapshot" ]
 then
     TEST_FILES="$TEST_FILES.snapshot"
@@ -64,8 +71,14 @@ else
     exit 1
 fi
 
-total=$(cat $TEST_FILES | wc -l)
+# Remove the skipped tests from list
-if [ "$total" -eq 0 ]
+for TEST in "${SKIP_LIST[@]}"
+do
+    ( sed -i "/$TEST/d" $TEST_FILES )
+done
+
+TOTAL=$(cat $TEST_FILES | wc -l)
+if [ "$TOTAL" -eq 0 ]
 then
     echo "$0: $TESTS: no test in test suite"
     exit 1
@@ -122,7 +135,7 @@ do
 
         if [ $status_code -eq 0 ]
         then
-            echo "[$tested/$total] $cmd_line: PASS"
+            echo "[$tested/$TOTAL] $cmd_line: PASS"
 
             cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
             ( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP )
@@ -138,7 +151,7 @@ do
 
     if [ $status_code -ne $error_code ]
     then
-        echo "[$tested/$total] $cmd_line: FAIL ($status_code)"
+        echo "[$tested/$TOTAL] $cmd_line: FAIL ($status_code)"
         cat $ENGINE_TEMP
 
         echo "$status_code: $test" >> $TEST_FAILED
@@ -150,7 +163,7 @@ do
 
         failed=$((failed+1))
     else
-        echo "[$tested/$total] $cmd_line: $PASS"
+        echo "[$tested/$TOTAL] $cmd_line: $PASS"
 
         echo "$test" >> $TEST_PASSED
 
@@ -162,14 +175,14 @@ done
 
 rm -f $ENGINE_TEMP
 
-ratio=$(echo $passed*100/$total | bc)
+ratio=$(echo $passed*100/$TOTAL | bc)
 
 if [ "$IS_SNAPSHOT" == true ]
 then
     ENGINE_ARGS="--snapshot $ENGINE_ARGS"
 fi
 
-echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $total total, $ratio% success"
+echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $TOTAL total, $ratio% success"
 
 if [ $failed -ne 0 ]
 then