diff --git a/.travis.yml b/.travis.yml index 5c96550a9..489695321 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ matrix: packages: - gcc-5 - gcc-5-multilib - env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600 + env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-O2,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600 - compiler: gcc-5 addons: apt: @@ -36,9 +36,8 @@ matrix: packages: - gcc-5 - gcc-5-multilib - env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600 + env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600 allow_failures: - - compiler: gcc-5 - env: OPTS="--check-pylint" fast_finish: true diff --git a/tools/run-tests.py b/tools/run-tests.py index 568e2d2ef..948fe3217 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -25,6 +25,7 @@ OUTPUT_DIR = path.join(PROJECT_DIR, 'build', 'tests') parser = argparse.ArgumentParser() parser.add_argument('--toolchain', action='store', default='', help='Add toolchain file') parser.add_argument('--buildoptions', action='store', default='', help='Add a comma separated list of extra build options to each test') +parser.add_argument('--skip-list', action='store', default='', help='Add a comma separated list of patterns of the excluded JS-tests') parser.add_argument('--outdir', action='store', default=OUTPUT_DIR, help='Specify output directory (default: %(default)s)') parser.add_argument('--check-signed-off', action='store_true', default=False, help='Run signed-off check') parser.add_argument('--check-signed-off-tolerant', action='store_true', default=False, help='Run signed-off check in tolerant mode') @@ -184,6 +185,9 @@ def run_jerry_tests(): break test_cmd = [TEST_RUNNER_SCRIPT, get_binary_path(job.out_dir), JERRY_TESTS_DIR] + if script_args.skip_list: + test_cmd.append("--skip-list=" + script_args.skip_list) + if job.test_args: test_cmd.extend(job.test_args) @@ -207,6 +211,9 @@ def run_jerry_test_suite(): else: test_cmd.append(JERRY_TEST_SUITE_ES51_LIST) + if script_args.skip_list: + test_cmd.append("--skip-list=" + script_args.skip_list) + if job.test_args: test_cmd.extend(job.test_args) diff --git a/tools/runners/run-test-suite.sh b/tools/runners/run-test-suite.sh index 6137afef4..460d532c7 100755 --- a/tools/runners/run-test-suite.sh +++ b/tools/runners/run-test-suite.sh @@ -15,7 +15,7 @@ # limitations under the License. # Usage: -# ./tools/runners/run-test-suite.sh ENGINE TESTS [--snapshot] ENGINE_ARGS.... +# ./tools/runners/run-test-suite.sh ENGINE TESTS [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS.... TIMEOUT=${TIMEOUT:=5} @@ -32,6 +32,13 @@ TEST_FILES=$OUTPUT_DIR/$TESTS_BASENAME.files TEST_FAILED=$OUTPUT_DIR/$TESTS_BASENAME.failed TEST_PASSED=$OUTPUT_DIR/$TESTS_BASENAME.passed +if [[ "$1" =~ ^--skip-list=.* ]] +then + SKIP_LIST=${1#--skip-list=} + SKIP_LIST=(${SKIP_LIST//,/ }) + shift +fi + if [ "$1" == "--snapshot" ] then TEST_FILES="$TEST_FILES.snapshot" @@ -64,8 +71,14 @@ else exit 1 fi -total=$(cat $TEST_FILES | wc -l) -if [ "$total" -eq 0 ] +# Remove the skipped tests from list +for TEST in "${SKIP_LIST[@]}" +do + ( sed -i "/$TEST/d" $TEST_FILES ) +done + +TOTAL=$(cat $TEST_FILES | wc -l) +if [ "$TOTAL" -eq 0 ] then echo "$0: $TESTS: no test in test suite" exit 1 @@ -122,7 +135,7 @@ do if [ $status_code -eq 0 ] then - echo "[$tested/$total] $cmd_line: PASS" + echo "[$tested/$TOTAL] $cmd_line: PASS" cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP" ( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP ) @@ -138,7 +151,7 @@ do if [ $status_code -ne $error_code ] then - echo "[$tested/$total] $cmd_line: FAIL ($status_code)" + echo "[$tested/$TOTAL] $cmd_line: FAIL ($status_code)" cat $ENGINE_TEMP echo "$status_code: $test" >> $TEST_FAILED @@ -150,7 +163,7 @@ do failed=$((failed+1)) else - echo "[$tested/$total] $cmd_line: $PASS" + echo "[$tested/$TOTAL] $cmd_line: $PASS" echo "$test" >> $TEST_PASSED @@ -162,14 +175,14 @@ done rm -f $ENGINE_TEMP -ratio=$(echo $passed*100/$total | bc) +ratio=$(echo $passed*100/$TOTAL | bc) if [ "$IS_SNAPSHOT" == true ] then ENGINE_ARGS="--snapshot $ENGINE_ARGS" fi -echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $total total, $ratio% success" +echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $TOTAL total, $ratio% success" if [ $failed -ne 0 ] then