Make test runners support execution runtimes (#2360)

If the result of a build is not directly executable on the host
system or needs an execution runtime for any other reason (e.g.,
cross-compiled ARM binaries on Intel, emscripten-generated JS
files, binaries built with Valgrind support) then the current test
runners cannot work with them. This patch makes test runners
execute binaries via a runtime given in the `$RUNTIME` environment
variable (if it is specified).

JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu

.travis.yml

@@ -38,8 +38,8 @@ matrix:
     - env:
         - JOBNAME="Linux/ARM Build & Correctness Tests"
         - OPTS="--quiet --jerry-tests --jerry-test-suite --toolchain=cmake/toolchain_linux_armv7l.cmake"
+        - RUNTIME=qemu-arm-static
         - TIMEOUT=300
-      sudo: true # keep on sudo-enabled VM to allow qemu to register itself to binfmt_misc
       addons:
         apt:
           packages: [gcc-arm-linux-gnueabihf, libc6-dev-armhf-cross, qemu-user-static]

tools/runners/run-test-suite-test262.sh

@@ -24,6 +24,12 @@ if [ $? -ne 0 ]
 then
     TIMEOUT_CMD=`which gtimeout`
 fi
+if [ -z "${RUNTIME}" ]
+then
+  COMMAND="${TIMEOUT_CMD} ${TIMEOUT} ${ENGINE}"
+else
+  COMMAND="${TIMEOUT_CMD} ${TIMEOUT} ${RUNTIME} ${ENGINE}"
+fi
 
 if [ $# -lt 2 ]
 then
@@ -56,7 +62,7 @@ rm -f "${PATH_TO_TEST262}/test/suite/ch15/15.9/15.9.3/S15.9.3.1_A5_T6.js"
 
 echo "Starting test262 testing for ${ENGINE}. Running test262 may take a several minutes."
 
-python2 "${PATH_TO_TEST262}"/tools/packaging/test262.py --command "${TIMEOUT_CMD} ${TIMEOUT} ${ENGINE}" \
+python2 "${PATH_TO_TEST262}"/tools/packaging/test262.py --command "${COMMAND}" \
                                                         --tests="${PATH_TO_TEST262}" --summary \
                                                         &> "${REPORT_PATH}"
 TEST262_EXIT_CODE=$?

tools/runners/run-test-suite.sh

@@ -148,23 +148,23 @@ do
         # Testing snapshot
         SNAPSHOT_TEMP=`mktemp $(basename -s .js $test).snapshot.XXXXXXXXXX`
 
-        cmd_line="${SNAPSHOT_TOOL#$ROOT_DIR} generate -o $SNAPSHOT_TEMP ${full_test#$ROOT_DIR}"
+        cmd_line="$RUNTIME ${SNAPSHOT_TOOL#$ROOT_DIR} generate -o $SNAPSHOT_TEMP ${full_test#$ROOT_DIR}"
-        $TIMEOUT_CMD $TIMEOUT $SNAPSHOT_TOOL generate -o $SNAPSHOT_TEMP $full_test &> $ENGINE_TEMP
+        $TIMEOUT_CMD $TIMEOUT $RUNTIME $SNAPSHOT_TOOL generate -o $SNAPSHOT_TEMP $full_test &> $ENGINE_TEMP
         status_code=$?
 
         if [ $status_code -eq 0 ]
         then
             test $VERBOSE && echo "[$tested/$TOTAL] $cmd_line: PASS"
 
-            cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
+            cmd_line="$RUNTIME ${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
-            $TIMEOUT_CMD $TIMEOUT $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP
+            $TIMEOUT_CMD $TIMEOUT $RUNTIME $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP
             status_code=$?
         fi
 
         rm -f $SNAPSHOT_TEMP
     else
-        cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${full_test#$ROOT_DIR}"
+        cmd_line="$RUNTIME ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${full_test#$ROOT_DIR}"
-        $TIMEOUT_CMD $TIMEOUT $ENGINE $ENGINE_ARGS $full_test &> $ENGINE_TEMP
+        $TIMEOUT_CMD $TIMEOUT $RUNTIME $ENGINE $ENGINE_ARGS $full_test &> $ENGINE_TEMP
         status_code=$?
     fi
 

tools/runners/run-unittests.sh

@@ -65,8 +65,8 @@ UNITTEST_TEMP=`mktemp unittest-out.XXXXXXXXXX`
 
 for unit_test in $UNITTESTS
 do
-    cmd_line="${unit_test#$ROOT_DIR}"
+    cmd_line="$RUNTIME ${unit_test#$ROOT_DIR}"
-    $unit_test &>$UNITTEST_TEMP
+    $RUNTIME $unit_test &>$UNITTEST_TEMP
     status_code=$?
 
     if [ $status_code -ne 0 ]