Akos Kiss
d7f95434e2
* Improve the name of the temporary snapshot files in test runner: Until now, test runner used a completely random filename for the temporary snapshot file when testing (saving to and executing from) snapshots. This patch makes the temp file contain part of the name of the original test file in order to make reading the logs (and thus, identifying failing tests) easier. * Add `--snapshot` to summary info of test runner if testing snapshot support: This helps reading the logs, especially when multiple tests run in parallel and get their output mixed. E.g., on the CI, all tests write the console at the same time and messages get interleaved. JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu
179 lines
4.2 KiB
Bash
Executable File
179 lines
4.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2015-2016 Samsung Electronics Co., Ltd.
|
|
# Copyright 2016 University of Szeged
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Usage:
|
|
# ./tools/runners/run-test-suite.sh ENGINE TESTS [--snapshot] ENGINE_ARGS....
|
|
|
|
TIMEOUT=${TIMEOUT:=5}
|
|
|
|
TEST_FILES=test.files
|
|
TEST_FAILED=test.failed
|
|
TEST_PASSED=test.passed
|
|
|
|
ENGINE="$1"
|
|
shift
|
|
|
|
TESTS="$1"
|
|
shift
|
|
|
|
if [ "$1" == "--snapshot" ]
|
|
then
|
|
TEST_FILES="snapshot.$TEST_FILES"
|
|
TEST_FAILED="snapshot.$TEST_FAILED"
|
|
TEST_PASSED="snapshot.$TEST_PASSED"
|
|
IS_SNAPSHOT=true;
|
|
shift
|
|
fi
|
|
|
|
ENGINE_ARGS="$@"
|
|
|
|
if [ ! -x $ENGINE ]
|
|
then
|
|
echo "$0: $ENGINE: not an executable"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -d $TESTS ]
|
|
then
|
|
TESTS_DIR=$TESTS
|
|
|
|
( cd $TESTS; find . -name "[^N]*.js" ) | sort > $TEST_FILES
|
|
elif [ -f $TESTS ]
|
|
then
|
|
TESTS_DIR=`dirname $TESTS`
|
|
|
|
grep -e '.js\s*$' $TESTS | sort > $TEST_FILES
|
|
else
|
|
echo "$0: $TESTS: not a test suite"
|
|
exit 1
|
|
fi
|
|
|
|
total=$(cat $TEST_FILES | wc -l)
|
|
if [ "$total" -eq 0 ]
|
|
then
|
|
echo "$0: $TESTS: no test in test suite"
|
|
exit 1
|
|
fi
|
|
|
|
rm -f $TEST_FAILED $TEST_PASSED
|
|
|
|
ROOT_DIR=""
|
|
CURRENT_DIR=`pwd`
|
|
PATH_STEP=2
|
|
while true
|
|
do
|
|
TMP_ROOT_DIR=`(echo "$CURRENT_DIR"; echo "$0"; echo "$ENGINE"; echo "$TESTS") | cut -f1-$PATH_STEP -d/ | uniq -d`
|
|
if [ -z "$TMP_ROOT_DIR" ]
|
|
then
|
|
break
|
|
else
|
|
ROOT_DIR="$TMP_ROOT_DIR"
|
|
fi
|
|
PATH_STEP=$((PATH_STEP+1))
|
|
done
|
|
if [ -n "$ROOT_DIR" ]
|
|
then
|
|
ROOT_DIR="$ROOT_DIR/"
|
|
fi
|
|
|
|
tested=1
|
|
failed=0
|
|
passed=0
|
|
|
|
ENGINE_TEMP=`mktemp engine-out.XXXXXXXXXX`
|
|
|
|
for test in `cat $TEST_FILES`
|
|
do
|
|
error_code=`echo $test | grep -e "^.\/fail\/[0-9]*\/" -o | cut -d / -f 3`
|
|
if [ "$error_code" = "" ]
|
|
then
|
|
PASS="PASS"
|
|
error_code=0
|
|
else
|
|
PASS="PASS (XFAIL)"
|
|
fi
|
|
|
|
full_test=$TESTS_DIR/${test#./}
|
|
|
|
if [ "$IS_SNAPSHOT" == true ]
|
|
then
|
|
# Testing snapshot
|
|
SNAPSHOT_TEMP=`mktemp $(basename -s .js $test).XXXXXXXXXX.snapshot`
|
|
|
|
cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --save-snapshot-for-global $SNAPSHOT_TEMP ${full_test#$ROOT_DIR}"
|
|
( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS --save-snapshot-for-global $SNAPSHOT_TEMP $full_test &> $ENGINE_TEMP )
|
|
status_code=$?
|
|
|
|
if [ $status_code -eq 0 ]
|
|
then
|
|
echo "[$tested/$total] $cmd_line: PASS"
|
|
|
|
cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
|
|
( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP )
|
|
status_code=$?
|
|
fi
|
|
|
|
rm -f $SNAPSHOT_TEMP
|
|
else
|
|
cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${full_test#$ROOT_DIR}"
|
|
( ulimit -t $TIMEOUT; $ENGINE $ENGINE_ARGS $full_test &> $ENGINE_TEMP )
|
|
status_code=$?
|
|
fi
|
|
|
|
if [ $status_code -ne $error_code ]
|
|
then
|
|
echo "[$tested/$total] $cmd_line: FAIL ($status_code)"
|
|
cat $ENGINE_TEMP
|
|
|
|
echo "$status_code: $test" >> $TEST_FAILED
|
|
echo "============================================" >> $TEST_FAILED
|
|
cat $ENGINE_TEMP >> $TEST_FAILED
|
|
echo "============================================" >> $TEST_FAILED
|
|
echo >> $TEST_FAILED
|
|
echo >> $TEST_FAILED
|
|
|
|
failed=$((failed+1))
|
|
else
|
|
echo "[$tested/$total] $cmd_line: $PASS"
|
|
|
|
echo "$test" >> $TEST_PASSED
|
|
|
|
passed=$((passed+1))
|
|
fi
|
|
|
|
tested=$((tested+1))
|
|
done
|
|
|
|
rm -f $ENGINE_TEMP
|
|
|
|
ratio=$(echo $passed*100/$total | bc)
|
|
|
|
if [ "$IS_SNAPSHOT" == true ]
|
|
then
|
|
ENGINE_ARGS="--snapshot $ENGINE_ARGS"
|
|
fi
|
|
|
|
echo "[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}: $passed PASS, $failed FAIL, $total total, $ratio% success"
|
|
|
|
if [ $failed -ne 0 ]
|
|
then
|
|
echo "$0: see $TEST_FAILED for details about failures"
|
|
exit 1
|
|
fi
|
|
|
|
exit 0
|