diff --git a/tests/jerry/fail/1/arguments-assignment-strict.js b/tests/jerry/fail/arguments-assignment-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-assignment-strict.js
rename to tests/jerry/fail/arguments-assignment-strict.js
diff --git a/tests/jerry/fail/1/arguments-catch-strict.js b/tests/jerry/fail/arguments-catch-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-catch-strict.js
rename to tests/jerry/fail/arguments-catch-strict.js
diff --git a/tests/jerry/fail/1/arguments-in-prop-set-param-list-strict.js b/tests/jerry/fail/arguments-in-prop-set-param-list-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-in-prop-set-param-list-strict.js
rename to tests/jerry/fail/arguments-in-prop-set-param-list-strict.js
diff --git a/tests/jerry/fail/1/arguments-in-var-decl-strict.js b/tests/jerry/fail/arguments-in-var-decl-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-in-var-decl-strict.js
rename to tests/jerry/fail/arguments-in-var-decl-strict.js
diff --git a/tests/jerry/fail/1/arguments-param-strict.js b/tests/jerry/fail/arguments-param-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-param-strict.js
rename to tests/jerry/fail/arguments-param-strict.js
diff --git a/tests/jerry/fail/1/arguments-postfix-strict.js b/tests/jerry/fail/arguments-postfix-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-postfix-strict.js
rename to tests/jerry/fail/arguments-postfix-strict.js
diff --git a/tests/jerry/fail/1/arguments-prefix-strict.js b/tests/jerry/fail/arguments-prefix-strict.js
similarity index 100%
rename from tests/jerry/fail/1/arguments-prefix-strict.js
rename to tests/jerry/fail/arguments-prefix-strict.js
diff --git a/tests/jerry/fail/1/delete-strict.js b/tests/jerry/fail/delete-strict.js
similarity index 100%
rename from tests/jerry/fail/1/delete-strict.js
rename to tests/jerry/fail/delete-strict.js
diff --git a/tests/jerry/fail/1/escape-sequences-invalid-hex.js b/tests/jerry/fail/escape-sequences-invalid-hex.js
similarity index 100%
rename from tests/jerry/fail/1/escape-sequences-invalid-hex.js
rename to tests/jerry/fail/escape-sequences-invalid-hex.js
diff --git a/tests/jerry/fail/1/escape-sequences-invalid-unicode.js b/tests/jerry/fail/escape-sequences-invalid-unicode.js
similarity index 100%
rename from tests/jerry/fail/1/escape-sequences-invalid-unicode.js
rename to tests/jerry/fail/escape-sequences-invalid-unicode.js
diff --git a/tests/jerry/fail/1/escape-sequences-invalid-variable.js b/tests/jerry/fail/escape-sequences-invalid-variable.js
similarity index 100%
rename from tests/jerry/fail/1/escape-sequences-invalid-variable.js
rename to tests/jerry/fail/escape-sequences-invalid-variable.js
diff --git a/tests/jerry/fail/1/eval-assignment-strict.js b/tests/jerry/fail/eval-assignment-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-assignment-strict.js
rename to tests/jerry/fail/eval-assignment-strict.js
diff --git a/tests/jerry/fail/1/eval-catch-strict.js b/tests/jerry/fail/eval-catch-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-catch-strict.js
rename to tests/jerry/fail/eval-catch-strict.js
diff --git a/tests/jerry/fail/1/eval-in-prop-set-param-list-strict.js b/tests/jerry/fail/eval-in-prop-set-param-list-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-in-prop-set-param-list-strict.js
rename to tests/jerry/fail/eval-in-prop-set-param-list-strict.js
diff --git a/tests/jerry/fail/1/eval-in-var-decl-strict.js b/tests/jerry/fail/eval-in-var-decl-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-in-var-decl-strict.js
rename to tests/jerry/fail/eval-in-var-decl-strict.js
diff --git a/tests/jerry/fail/1/eval-param-strict.js b/tests/jerry/fail/eval-param-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-param-strict.js
rename to tests/jerry/fail/eval-param-strict.js
diff --git a/tests/jerry/fail/1/eval-postfix-strict.js b/tests/jerry/fail/eval-postfix-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-postfix-strict.js
rename to tests/jerry/fail/eval-postfix-strict.js
diff --git a/tests/jerry/fail/1/eval-prefix-strict.js b/tests/jerry/fail/eval-prefix-strict.js
similarity index 100%
rename from tests/jerry/fail/1/eval-prefix-strict.js
rename to tests/jerry/fail/eval-prefix-strict.js
diff --git a/tests/jerry/fail/1/func-expr-strict.js b/tests/jerry/fail/func-expr-strict.js
similarity index 100%
rename from tests/jerry/fail/1/func-expr-strict.js
rename to tests/jerry/fail/func-expr-strict.js
diff --git a/tests/jerry/fail/1/labelled-statements-break-across-function.js b/tests/jerry/fail/labelled-statements-break-across-function.js
similarity index 100%
rename from tests/jerry/fail/1/labelled-statements-break-across-function.js
rename to tests/jerry/fail/labelled-statements-break-across-function.js
diff --git a/tests/jerry/fail/1/labelled-statements-duplicate-label.js b/tests/jerry/fail/labelled-statements-duplicate-label.js
similarity index 100%
rename from tests/jerry/fail/1/labelled-statements-duplicate-label.js
rename to tests/jerry/fail/labelled-statements-duplicate-label.js
diff --git a/tests/jerry/fail/1/labelled-statements-no-label.js b/tests/jerry/fail/labelled-statements-no-label.js
similarity index 100%
rename from tests/jerry/fail/1/labelled-statements-no-label.js
rename to tests/jerry/fail/labelled-statements-no-label.js
diff --git a/tests/jerry/fail/1/let-strict.js b/tests/jerry/fail/let-strict.js
similarity index 100%
rename from tests/jerry/fail/1/let-strict.js
rename to tests/jerry/fail/let-strict.js
diff --git a/tests/jerry/fail/1/object-get-data.js b/tests/jerry/fail/object-get-data.js
similarity index 100%
rename from tests/jerry/fail/1/object-get-data.js
rename to tests/jerry/fail/object-get-data.js
diff --git a/tests/jerry/fail/1/object-get-get.js b/tests/jerry/fail/object-get-get.js
similarity index 100%
rename from tests/jerry/fail/1/object-get-get.js
rename to tests/jerry/fail/object-get-get.js
diff --git a/tests/jerry/fail/1/object-several-prop-names-strict.js b/tests/jerry/fail/object-several-prop-names-strict.js
similarity index 100%
rename from tests/jerry/fail/1/object-several-prop-names-strict.js
rename to tests/jerry/fail/object-several-prop-names-strict.js
diff --git a/tests/jerry/fail/1/octal-strict.js b/tests/jerry/fail/octal-strict.js
similarity index 100%
rename from tests/jerry/fail/1/octal-strict.js
rename to tests/jerry/fail/octal-strict.js
diff --git a/tests/jerry/fail/1/param-duplication-strict.js b/tests/jerry/fail/param-duplication-strict.js
similarity index 100%
rename from tests/jerry/fail/1/param-duplication-strict.js
rename to tests/jerry/fail/param-duplication-strict.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1549.js b/tests/jerry/fail/regression-test-issue-1549.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1549.js
rename to tests/jerry/fail/regression-test-issue-1549.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1550.js b/tests/jerry/fail/regression-test-issue-1550.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1550.js
rename to tests/jerry/fail/regression-test-issue-1550.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1597.js b/tests/jerry/fail/regression-test-issue-1597.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1597.js
rename to tests/jerry/fail/regression-test-issue-1597.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1598.js b/tests/jerry/fail/regression-test-issue-1598.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1598.js
rename to tests/jerry/fail/regression-test-issue-1598.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1615.js b/tests/jerry/fail/regression-test-issue-1615.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1615.js
rename to tests/jerry/fail/regression-test-issue-1615.js
diff --git a/tests/jerry/fail/1/regression-test-issue-1624.js b/tests/jerry/fail/regression-test-issue-1624.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-1624.js
rename to tests/jerry/fail/regression-test-issue-1624.js
diff --git a/tests/jerry/fail/1/regression-test-issue-358.js b/tests/jerry/fail/regression-test-issue-358.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-358.js
rename to tests/jerry/fail/regression-test-issue-358.js
diff --git a/tests/jerry/fail/1/regression-test-issue-384.js b/tests/jerry/fail/regression-test-issue-384.js
similarity index 100%
rename from tests/jerry/fail/1/regression-test-issue-384.js
rename to tests/jerry/fail/regression-test-issue-384.js
diff --git a/tests/jerry/fail/1/throw-error-object.js b/tests/jerry/fail/throw-error-object.js
similarity index 100%
rename from tests/jerry/fail/1/throw-error-object.js
rename to tests/jerry/fail/throw-error-object.js
diff --git a/tests/jerry/fail/1/throw-number.js b/tests/jerry/fail/throw-number.js
similarity index 100%
rename from tests/jerry/fail/1/throw-number.js
rename to tests/jerry/fail/throw-number.js
diff --git a/tests/jerry/fail/1/throw-string.js b/tests/jerry/fail/throw-string.js
similarity index 100%
rename from tests/jerry/fail/1/throw-string.js
rename to tests/jerry/fail/throw-string.js
diff --git a/tests/jerry/fail/1/with-strict.js b/tests/jerry/fail/with-strict.js
similarity index 100%
rename from tests/jerry/fail/1/with-strict.js
rename to tests/jerry/fail/with-strict.js
diff --git a/tools/runners/run-test-suite.sh b/tools/runners/run-test-suite.sh
index 9c6d098ab..087514164 100755
--- a/tools/runners/run-test-suite.sh
+++ b/tools/runners/run-test-suite.sh
@@ -118,13 +118,13 @@ ENGINE_TEMP=`mktemp engine-out.XXXXXXXXXX`
 
 for test in `cat $TEST_FILES`
 do
-    error_code=`echo $test | grep -e "^.\/fail\/[0-9]*\/" -o | cut -d / -f 3`
-    if [ "$error_code" = "" ]
+    if [[ $test =~ ^\.\/fail\/ ]]
     then
-        PASS="PASS"
-        error_code=0
-    else
+        error_code=1
         PASS="PASS (XFAIL)"
+    else
+        error_code=0
+        PASS="PASS"
     fi
 
     full_test=$TESTS_DIR/${test#./}