Add quiet mode for test runner(s) (#2228)

In one of the previous PR we have encountered a problem,
where the Travis cuts off the test execution
as the generated log file is too big.

By adding a quiet mode for the test runners, we will
only report the failing tests.

JerryScript-DCO-1.0-Signed-off-by: Peter Gal pgal.u-szeged@partner.samsung.com

tools/run-tests.py

@@ -135,6 +135,8 @@ def get_arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument('--toolchain', metavar='FILE',
                         help='Add toolchain file')
+    parser.add_argument('-q', '--quiet', action='store_true',
+                        help='Only print out failing tests')
     parser.add_argument('--buildoptions', metavar='LIST',
                         help='Add a comma separated list of extra build options to each test')
     parser.add_argument('--skip-list', metavar='LIST',
@@ -266,6 +268,9 @@ def run_jerry_tests(options):
         if options.skip_list:
             skip_list.append(options.skip_list)
 
+        if options.quiet:
+            test_cmd.append("-q")
+
         if skip_list:
             test_cmd.append("--skip-list=" + ",".join(skip_list))
 
@@ -292,6 +297,9 @@ def run_jerry_test_suite(options):
         else:
             test_cmd.append(settings.JERRY_TEST_SUITE_ES51_LIST)
 
+        if options.quiet:
+            test_cmd.append("-q")
+
         if options.skip_list:
             test_cmd.append("--skip-list=" + options.skip_list)
 
@@ -331,7 +339,8 @@ def run_unittests(options):
 
         ret_test |= run_check([
             settings.UNITTEST_RUNNER_SCRIPT,
-            bin_dir_path
+            bin_dir_path,
+            "-q" if options.quiet else "",
         ])
 
     return ret_build | ret_test

tools/runners/run-test-suite.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Usage:
-#       ./tools/runners/run-test-suite.sh ENGINE TESTS [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS....
+#       ./tools/runners/run-test-suite.sh ENGINE TESTS [-q] [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS....
 
 TIMEOUT=${TIMEOUT:=5}
 TIMEOUT_CMD=`which timeout`
@@ -37,6 +37,13 @@ TEST_FILES=$OUTPUT_DIR/$TESTS_BASENAME.files
 TEST_FAILED=$OUTPUT_DIR/$TESTS_BASENAME.failed
 TEST_PASSED=$OUTPUT_DIR/$TESTS_BASENAME.passed
 
+VERBOSE=1
+if [[ "$1" == "-q" ]]
+then
+    unset VERBOSE
+    shift
+fi
+
 if [[ "$1" =~ ^--skip-list=.* ]]
 then
     SKIP_LIST=${1#--skip-list=}
@@ -147,7 +154,7 @@ do
 
         if [ $status_code -eq 0 ]
         then
-            echo "[$tested/$TOTAL] $cmd_line: PASS"
+            test $VERBOSE && echo "[$tested/$TOTAL] $cmd_line: PASS"
 
             cmd_line="${ENGINE#$ROOT_DIR} $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP"
             $TIMEOUT_CMD $TIMEOUT $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP
@@ -175,7 +182,7 @@ do
 
         failed=$((failed+1))
     else
-        echo "[$tested/$TOTAL] $cmd_line: $PASS"
+        test $VERBOSE && echo "[$tested/$TOTAL] $cmd_line: $PASS"
 
         echo "$test" >> $TEST_PASSED
 

tools/runners/run-unittests.sh

@@ -17,6 +17,13 @@
 DIR="$1"
 shift
 
+VERBOSE=1
+if [ "$1" == "-q" ]
+then
+    unset VERBOSE
+    shift
+fi
+
 UNITTEST_ERROR=$DIR/unittests.failed
 UNITTEST_OK=$DIR/unittests.passed
 
@@ -76,7 +83,7 @@ do
 
         failed=$((failed+1))
     else
-        echo "[$tested/$total] $cmd_line: PASS"
+        test $VERBOSE && echo "[$tested/$total] $cmd_line: PASS"
 
         echo "$unit_test" >> $UNITTEST_OK