YourWishes/jerryscript
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow to skip certain JS-tests under tests/jerry/ and tests/jerry-test-suite/ (#1632)

Browse Source 
Add `--skip-list` option to tools/run-tests.py.

Additional modifications:
 - The test `parser-oom.js` doesn't work when system-allocator is enabled,
   hence we skip it in these jobs.
 - The sanitizer-tasks become mandatory.

JerryScript-DCO-1.0-Signed-off-by: Zsolt Borbély zsborbely.u-szeged@partner.samsung.com
This commit is contained in:
Zsolt Borbély
2017-03-02 15:58:23 +01:00
committed by GitHub
parent 92f74f3cf5
commit f780f2b8dc
3 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -3
View File
@@ -27,7 +27,7 @@ matrix:
packages:
- gcc-5
- gcc-5-multilib
env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600
env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-O2,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true TIMEOUT=600
- compiler: gcc-5
addons:
apt:
@@ -36,9 +36,8 @@ matrix:
packages:
- gcc-5
- gcc-5-multilib
env: OPTS="--jerry-tests --jerry-test-suite --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-g,--jerry-libc=off,--static-link=off,--strip=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600
env: OPTS="--jerry-tests --jerry-test-suite --skip-list=parser-oom.js --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--debug,--jerry-libc=off,--static-link=off,--system-allocator=on,--linker-flag=-fuse-ld=gold" UBSAN_OPTIONS=print_stacktrace=1 TIMEOUT=600
allow_failures:
- compiler: gcc-5
- env: OPTS="--check-pylint"
fast_finish: true
tools/run-tests.py
+7
View File
@@ -25,6 +25,7 @@ OUTPUT_DIR = path.join(PROJECT_DIR, 'build', 'tests')
parser = argparse.ArgumentParser()
parser.add_argument('--toolchain', action='store', default='', help='Add toolchain file')
parser.add_argument('--buildoptions', action='store', default='', help='Add a comma separated list of extra build options to each test')
parser.add_argument('--skip-list', action='store', default='', help='Add a comma separated list of patterns of the excluded JS-tests')
parser.add_argument('--outdir', action='store', default=OUTPUT_DIR, help='Specify output directory (default: %(default)s)')
parser.add_argument('--check-signed-off', action='store_true', default=False, help='Run signed-off check')
parser.add_argument('--check-signed-off-tolerant', action='store_true', default=False, help='Run signed-off check in tolerant mode')
@@ -184,6 +185,9 @@ def run_jerry_tests():
break
test_cmd = [TEST_RUNNER_SCRIPT, get_binary_path(job.out_dir), JERRY_TESTS_DIR]
if script_args.skip_list:
test_cmd.append("--skip-list=" + script_args.skip_list)
if job.test_args:
test_cmd.extend(job.test_args)
@@ -207,6 +211,9 @@ def run_jerry_test_suite():
else:
test_cmd.append(JERRY_TEST_SUITE_ES51_LIST)
if script_args.skip_list:
test_cmd.append("--skip-list=" + script_args.skip_list)
if job.test_args:
test_cmd.extend(job.test_args)
tools/runners/run-test-suite.sh
+21 -8
View File
@@ -15,7 +15,7 @@
# limitations under the License.
# Usage:
# ./tools/runners/run-test-suite.sh ENGINE TESTS [--snapshot] ENGINE_ARGS....
# ./tools/runners/run-test-suite.sh ENGINE TESTS [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS....
TIMEOUT=${TIMEOUT:=5}
@@ -32,6 +32,13 @@ TEST_FILES=$OUTPUT_DIR/$TESTS_BASENAME.files
TEST_FAILED=$OUTPUT_DIR/$TESTS_BASENAME.failed
TEST_PASSED=$OUTPUT_DIR/$TESTS_BASENAME.passed
if [[ "$1" =~ ^--skip-list=.* ]]
then
SKIP_LIST=${1#--skip-list=}
SKIP_LIST=(${SKIP_LIST//,/ })
shift
fi
if [ "$1" == "--snapshot" ]
then
TEST_FILES="$TEST_FILES.snapshot"
@@ -64,8 +71,14 @@ else
exit 1
fi
total=$(cat $TEST_FILES | wc -l)
if [ "$total" -eq 0 ]
# Remove the skipped tests from list
for TEST in "${SKIP_LIST[@]}"
do
( sed -i "/$TEST/d" $TEST_FILES )
done
TOTAL=$(cat $TEST_FILES | wc -l)
if [ "$TOTAL" -eq 0 ]
then
echo "$0: $TESTS: no test in test suite"
exit 1
@@ -122,7 +135,7 @@ do
if [ $status_code -eq 0 ]
then
echo "[$tested/$total] $cmd_line: PASS"
echo "[$tested/$TOTAL] $cmd_line: PASS"
cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP )
@@ -138,7 +151,7 @@ do
if [ $status_code -ne $error_code ]
then
echo "[$tested/$total] $cmd_line: FAIL ($status_code)"
echo "[$tested/$TOTAL] $cmd_line: FAIL ($status_code)"
cat $ENGINE_TEMP
echo "$status_code: $test" >> $TEST_FAILED
@@ -150,7 +163,7 @@ do
failed=$((failed+1))
else
echo "[$tested/$total] $cmd_line: $PASS"
echo "[$tested/$TOTAL] $cmd_line: $PASS"
echo "$test" >> $TEST_PASSED
@@ -162,14 +175,14 @@ done
rm -f $ENGINE_TEMP
ratio=$(echo $passed*100/$total | bc)
ratio=$(echo $passed*100/$TOTAL | bc)
if [ "$IS_SNAPSHOT" == true ]
then
ENGINE_ARGS="--snapshot $ENGINE_ARGS"
fi
echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $total total, $ratio% success"
echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $TOTAL total, $ratio% success"
if [ $failed -ne 0 ]
then