Ensure that the test version of the command line tool is stable for benchmarking (#2076)

Some benchmark suites contain test cases that have nonreproducible
behaviour. This is mostly caused by relying on "now" when dealing
with dates or timestamps, instead of using a fixed moment. (A
notorious example is the crypto-aes.js test case of the sunspider
bechmark suite, where the heap memory consumption can vary between
34K-41K heap because of using `(new Date()).getTime()`.)

This commit renames the jerry-minimal command line tool to
jerry-test (to better reflect its purpose) and adds extra code,
which intercepts some calls to libc (`gettimeofday`, `rand`) and
pins their results to some fixed values. This makes the tool
useless in a general case but ensures stable results when
benchmarking -- for which it is mostly used.

As a side effect, the commit also changes jerry-libc by making all
libc functions weak symbols to allow their override from
application code.

JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu

CMakeLists.txt

@@ -39,7 +39,7 @@ endif()
 
 # Optional components
 set(JERRY_CMDLINE           ON  CACHE BOOL "Build jerry command line tool?")
-set(JERRY_CMDLINE_MINIMAL   OFF CACHE BOOL "Build jerry minimal command line tool?")
+set(JERRY_CMDLINE_TEST      OFF CACHE BOOL "Build jerry test command line tool?")
 set(JERRY_CMDLINE_SNAPSHOT  OFF CACHE BOOL "Build jerry snapshot command line tool?")
 set(JERRY_PORT_DEFAULT      ON  CACHE BOOL "Build default jerry port implementation?")
 set(JERRY_EXT               ON  CACHE BOOL "Build jerry-ext?")
@@ -58,7 +58,7 @@ set(ENABLE_STRIP       ON  CACHE BOOL "Enable stripping all symbols from release
 set(FEATURE_INIT_FINI OFF CACHE BOOL "Enable init/fini arrays?")
 
 # Option overrides
-if(JERRY_CMDLINE OR JERRY_CMDLINE_MINIMAL OR JERRY_CMDLINE_SNAPSHOT OR UNITTESTS OR DOCTESTS)
+if(JERRY_CMDLINE OR JERRY_CMDLINE_TEST OR JERRY_CMDLINE_SNAPSHOT OR UNITTESTS OR DOCTESTS)
   set(JERRY_PORT_DEFAULT ON)
 
   set(JERRY_PORT_DEFAULT_MESSAGE " (FORCED BY CMDLINE OR TESTS)")
@@ -103,7 +103,7 @@ message(STATUS "ENABLE_LTO                " ${ENABLE_LTO} ${ENABLE_LTO_MESSAGE})
 message(STATUS "ENABLE_STATIC_LINK        " ${ENABLE_STATIC_LINK} ${ENABLE_STATIC_LINK_MESSAGE})
 message(STATUS "ENABLE_STRIP              " ${ENABLE_STRIP} ${ENABLE_STRIP_MESSAGE})
 message(STATUS "JERRY_CMDLINE             " ${JERRY_CMDLINE})
-message(STATUS "JERRY_CMDLINE_MINIMAL     " ${JERRY_CMDLINE_MINIMAL})
+message(STATUS "JERRY_CMDLINE_TEST        " ${JERRY_CMDLINE_TEST})
 message(STATUS "JERRY_CMDLINE_SNAPSHOT    " ${JERRY_CMDLINE_SNAPSHOT})
 message(STATUS "JERRY_PORT_DEFAULT        " ${JERRY_PORT_DEFAULT} ${JERRY_PORT_DEFAULT_MESSAGE})
 message(STATUS "JERRY_EXT                 " ${JERRY_EXT} ${JERRY_EXT_MESSAGE})
@@ -270,7 +270,7 @@ if(JERRY_EXT)
 endif()
 
 # Jerry command line tool
-if(JERRY_CMDLINE OR JERRY_CMDLINE_MINIMAL OR JERRY_CMDLINE_SNAPSHOT)
+if(JERRY_CMDLINE OR JERRY_CMDLINE_TEST OR JERRY_CMDLINE_SNAPSHOT)
   add_subdirectory(jerry-main)
 endif()
 

jerry-libc/jerry-libc-printf.c

@@ -469,7 +469,7 @@ libc_printf_write_u_o_x_X (FILE *stream, /**< stream pointer */
  *
  * @return number of characters printed
  */
-int
+int __attr_weak___
 vfprintf (FILE *stream, /**< stream pointer */
           const char *format, /**< format string */
           va_list args) /**< arguments */
@@ -719,7 +719,7 @@ vfprintf (FILE *stream, /**< stream pointer */
  *
  * @return number of characters printed
  */
-int
+int __attr_weak___
 fprintf (FILE *stream, /**< stream pointer */
          const char *format, /**< format string */
          ...) /**< parameters' values */
@@ -740,7 +740,7 @@ fprintf (FILE *stream, /**< stream pointer */
  *
  * @return number of characters printed
  */
-int
+int __attr_weak___
 printf (const char *format, /**< format string */
         ...) /**< parameters' values */
 {

jerry-libc/jerry-libc.c

@@ -56,7 +56,7 @@ CALL_PRAGMA (GCC optimize ("-fno-tree-loop-distribute-patterns"))
  *
  * @return @a s
  */
-void * __attr_used___
+void * __attr_weak___ __attr_used___
 memset (void *s, /**< area to set values in */
         int c, /**< value to set */
         size_t n) /**< area size */
@@ -77,7 +77,7 @@ memset (void *s, /**< area to set values in */
  *         <0, if first area's content is lexicographically less, than second area's content;
  *         >0, otherwise
  */
-int
+int __attr_weak___
 memcmp (const void *s1, /**< first area */
         const void *s2, /**< second area */
         size_t n) /**< area size */
@@ -100,7 +100,7 @@ memcmp (const void *s1, /**< first area */
  *
  * @return the dest pointer's value
  */
-void *  __attr_used___
+void * __attr_weak___ __attr_used___
 memcpy (void *s1, /**< destination */
         const void *s2, /**< source */
         size_t n) /**< bytes number */
@@ -139,7 +139,7 @@ memcpy (void *s1, /**< destination */
  *
  * @return the dest pointer's value
  */
-void * __attr_used___
+void * __attr_weak___ __attr_used___
 memmove (void *s1, /**< destination */
          const void *s2, /**< source */
          size_t n) /**< bytes number */
@@ -182,7 +182,7 @@ CALL_PRAGMA (GCC diagnostic pop)
  * @return an integer less than, equal to, or greater than zero if s1 is found, respectively,
  *         to be less than, to match, or be greater than s2.
  */
-int
+int __attr_weak___
 strcmp (const char *s1, /**< first string */
         const char *s2) /**< second string */
 {
@@ -205,7 +205,7 @@ strcmp (const char *s1, /**< first string */
  * @return an integer less than, equal to, or greater than zero if the first n character of s1 is found, respectively,
  *         to be less than, to match, or be greater than the first n character of s2.
  */
-int
+int __attr_weak___
 strncmp (const char *s1, /**< first string */
          const char *s2, /**< second string */
          size_t n) /**< maximum number of characters to compare */
@@ -234,7 +234,7 @@ strncmp (const char *s1, /**< first string */
  *
  * @return a pointer to the destination string dest.
  */
-char * __attr_used___
+char * __attr_weak___ __attr_used___
 strncpy (char *dest, /**< destination string */
          const char *src, /**< source string */
          size_t n) /**< maximum number of characters to copy */
@@ -258,7 +258,7 @@ strncpy (char *dest, /**< destination string */
  *
  * @return the length.
  */
-size_t
+size_t __attr_weak___
 strlen (const char *s) /**< string */
 {
   size_t i = 0;
@@ -278,7 +278,7 @@ strlen (const char *s) /**< string */
  *
  * @return integer in range [0; RAND_MAX]
  */
-int
+int __attr_weak___
 rand (void)
 {
   uint32_t intermediate = libc_random_gen_state[0] ^ (libc_random_gen_state[0] << 11);
@@ -297,7 +297,7 @@ rand (void)
 /**
  * Initialize pseudo-random number generator with the specified seed value
  */
-void
+void __attr_weak___
 srand (unsigned int seed) /**< new seed */
 {
   libc_random_gen_state[0] = (uint32_t) ((seed * 14316555781)
@@ -325,7 +325,7 @@ srand (unsigned int seed) /**< new seed */
  *
  * @return the integer value of str.
  */
-long int
+long int __attr_weak___
 strtol (const char *nptr, /**< string representation of an integer number */
         char **endptr, /**< [out] the address of the first non-number character */
         int base) /**< numerical base or radix (MUST be 10) */

jerry-libc/target/posix/jerry-libc-target.c

@@ -48,7 +48,7 @@ long int syscall_3 (long int syscall_no, long int arg1, long int arg2, long int
 /**
  * Exit - cause normal process termination with specified status code
  */
-void __attr_noreturn___ __attr_used___
+void __attr_weak___ __attr_noreturn___ __attr_used___
 exit (int status) /**< status code */
 {
 #ifdef ENABLE_INIT_FINI
@@ -71,7 +71,7 @@ exit (int status) /**< status code */
  * Abort current process, producing an abnormal program termination.
  * The function raises the SIGABRT signal.
  */
-void __attr_noreturn___ __attr_used___
+void __attr_weak___ __attr_noreturn___ __attr_used___
 abort (void)
 {
   syscall_1 (SYSCALL_NO (close), (long int) stdin);
@@ -92,7 +92,7 @@ abort (void)
  * @return 0 - upon successful completion,
  *         non-zero value - otherwise.
  */
-int __attr_used___
+int __attr_weak___ __attr_used___
 raise (int sig) /**< signal number */
 {
   return (int) syscall_2 (SYSCALL_NO (kill), syscall_0 (SYSCALL_NO (getpid)), sig);
@@ -104,7 +104,7 @@ raise (int sig) /**< signal number */
  * @return FILE pointer - upon successful completion,
  *         NULL - otherwise.
  */
-FILE *
+FILE * __attr_weak___
 fopen (const char *path, /**< file path */
        const char *mode) /**< file open mode */
 {
@@ -193,7 +193,7 @@ fopen (const char *path, /**< file path */
  * @return 0 - upon successful completion,
  *         non-zero value - otherwise.
  */
-int
+int __attr_weak___
 fclose (FILE *fp) /**< stream pointer */
 {
   syscall_2 (SYSCALL_NO (close), (long int) fp, 0);
@@ -206,7 +206,7 @@ fclose (FILE *fp) /**< stream pointer */
  *
  * @return number of elements read
  */
-size_t
+size_t __attr_weak___
 fread (void *ptr, /**< address of buffer to read to */
        size_t size, /**< size of elements to read */
        size_t nmemb, /**< number of elements to read */
@@ -239,7 +239,7 @@ fread (void *ptr, /**< address of buffer to read to */
  *
  * @return number of elements written
  */
-size_t
+size_t __attr_weak___
 fwrite (const void *ptr, /**< data to write */
         size_t size, /**< size of elements to write */
         size_t nmemb, /**< number of elements */
@@ -271,7 +271,7 @@ fwrite (const void *ptr, /**< data to write */
  *
  * @return 0 if success, -1 otherwise
  */
-int
+int __attr_weak___
 gettimeofday (void *tp,  /**< struct timeval */
               void *tzp) /**< struct timezone */
 {

jerry-main/CMakeLists.txt

@@ -64,9 +64,9 @@ if(JERRY_CMDLINE)
   target_link_libraries("jerry" jerry-ext jerry-port-default)
 endif()
 
-if(JERRY_CMDLINE_MINIMAL)
+if(JERRY_CMDLINE_TEST)
-  jerry_create_executable("jerry-minimal" "main-unix-minimal.c")
+  jerry_create_executable("jerry-test" "main-unix-test.c" "benchmarking.c")
-  target_link_libraries("jerry-minimal" jerry-port-default-minimal)
+  target_link_libraries("jerry-test" jerry-port-default-minimal)
 endif()
 
 if(JERRY_CMDLINE_SNAPSHOT)

jerry-main/benchmarking.c

@@ -0,0 +1,63 @@
+/* Copyright JS Foundation and other contributors, http://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This source contains libc overrides for the sake of stable benchmarking. If
+ * building a binary for the purpose of benchmarking, the object compiled from
+ * this source is to be injected into the list of objects-to-be-linked before
+ * the list of libraries-to-be-linked to ensure that the linker picks up these
+ * implementations.
+ */
+
+ #ifdef __GNUC__
+/*
+ * Note:
+ *      This is nasty and dangerous. However, we only need the timeval structure
+ *      from sys/time.h. Unfortunately, the same header also declares
+ *      gettimeofday, which has different declarations on different platforms
+ *      (e.g., macOS, Linux). So, instead of #ifdef'ing for platforms, we simply
+ *      tweak the header to declare another function. Don't try this at home.
+ */
+#define gettimeofday __prevent_conflicting_gettimeofday_declarations__
+#include <sys/time.h>
+#undef gettimeofday
+
+int gettimeofday (struct timeval *, void *);
+
+/**
+ * Useless but stable gettimeofday implementation. Returns Epoch. Ensures that
+ * test cases relying on "now" yield stable results.
+ */
+int gettimeofday (struct timeval *tv,
+                  void *tz)
+{
+  (void) tz;
+  tv->tv_sec = 0;
+  tv->tv_usec = 0;
+  return 0;
+} /* gettimeofday */
+#endif /* __GNUC__ */
+
+
+int rand (void);
+
+/**
+ * Useless but stable rand implementation. Returns 4. Ensures that test cases
+ * relying on randomness yield stable results.
+ */
+int rand (void)
+{
+  return 4; /* Chosen by fair dice roll. Guaranteed to be random. */
+} /* rand */

tools/build.py

@@ -72,8 +72,6 @@ def get_arguments():
                         help='Allowed N build jobs at once (default: %(default)s)')
     parser.add_argument('--jerry-cmdline', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper,
                         help='build jerry command line tool (%(choices)s; default: %(default)s)')
-    parser.add_argument('--jerry-cmdline-minimal', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper,
-                        help='build minimal version of the jerry command line tool (%(choices)s; default: %(default)s)')
     parser.add_argument('--jerry-cmdline-snapshot', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper,
                         help='build snapshot command line tool (%(choices)s; default: %(default)s)')
     parser.add_argument('--jerry-debugger', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper,
@@ -118,6 +116,9 @@ def get_arguments():
                         help='enable VM execution stopping (%(choices)s; default: %(default)s)')
 
     devgroup = parser.add_argument_group('developer options')
+    devgroup.add_argument('--jerry-cmdline-test', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper,
+                          help=devhelp('build test version of the jerry command line tool '
+                                       '(%(choices)s; default: %(default)s)'))
     devgroup.add_argument('--link-map', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper,
                           help=devhelp('enable the generation of a link map file for jerry command line tool '
                                        '(%(choices)s; default: %(default)s)'))
@@ -151,7 +152,7 @@ def generate_build_options(arguments):
     build_options.append('-DFEATURE_CPOINTER_32_BIT=%s' % arguments.cpointer_32bit)
     build_options.append('-DFEATURE_ERROR_MESSAGES=%s' % arguments.error_messages)
     build_options.append('-DJERRY_CMDLINE=%s' % arguments.jerry_cmdline)
-    build_options.append('-DJERRY_CMDLINE_MINIMAL=%s' % arguments.jerry_cmdline_minimal)
+    build_options.append('-DJERRY_CMDLINE_TEST=%s' % arguments.jerry_cmdline_test)
     build_options.append('-DJERRY_CMDLINE_SNAPSHOT=%s' % arguments.jerry_cmdline_snapshot)
     build_options.append('-DJERRY_PORT_DEFAULT=%s' % arguments.jerry_port_default)
     build_options.append('-DJERRY_EXT=%s' % arguments.jerry_ext)

tools/run-tests.py

@@ -122,8 +122,8 @@ JERRY_BUILDOPTIONS = [
             ['--jerry-libc=off', '--compile-flag=-m32', '--cpointer-32bit=on', '--system-allocator=on']),
     Options('buildoption_test-external_context',
             ['--jerry-libc=off', '--external-context=on']),
-    Options('buildoption_test-cmdline_minimal',
+    Options('buildoption_test-cmdline_test',
-            ['--jerry-cmdline-minimal=on']),
+            ['--jerry-cmdline-test=on']),
     Options('buildoption_test-cmdline_snapshot',
             ['--jerry-cmdline-snapshot=on']),
 ]