From a0db3ee5b302a6574b229cc807869afe7107c872 Mon Sep 17 00:00:00 2001 From: Akos Kiss Date: Thu, 16 Nov 2017 12:36:58 +0100 Subject: [PATCH] Ensure that the test version of the command line tool is stable for benchmarking (#2076) Some benchmark suites contain test cases that have nonreproducible behaviour. This is mostly caused by relying on "now" when dealing with dates or timestamps, instead of using a fixed moment. (A notorious example is the crypto-aes.js test case of the sunspider bechmark suite, where the heap memory consumption can vary between 34K-41K heap because of using `(new Date()).getTime()`.) This commit renames the jerry-minimal command line tool to jerry-test (to better reflect its purpose) and adds extra code, which intercepts some calls to libc (`gettimeofday`, `rand`) and pins their results to some fixed values. This makes the tool useless in a general case but ensures stable results when benchmarking -- for which it is mostly used. As a side effect, the commit also changes jerry-libc by making all libc functions weak symbols to allow their override from application code. JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu --- CMakeLists.txt | 8 +-- jerry-libc/jerry-libc-printf.c | 6 +- jerry-libc/jerry-libc.c | 22 +++---- jerry-libc/target/posix/jerry-libc-target.c | 16 ++--- jerry-main/CMakeLists.txt | 6 +- jerry-main/benchmarking.c | 63 +++++++++++++++++++ .../{main-unix-minimal.c => main-unix-test.c} | 0 tools/build.py | 7 ++- tools/run-tests.py | 4 +- 9 files changed, 98 insertions(+), 34 deletions(-) create mode 100644 jerry-main/benchmarking.c rename jerry-main/{main-unix-minimal.c => main-unix-test.c} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32f9f89d4..bab261295 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ endif() # Optional components set(JERRY_CMDLINE ON CACHE BOOL "Build jerry command line tool?") -set(JERRY_CMDLINE_MINIMAL OFF CACHE BOOL "Build jerry minimal command line tool?") +set(JERRY_CMDLINE_TEST OFF CACHE BOOL "Build jerry test command line tool?") set(JERRY_CMDLINE_SNAPSHOT OFF CACHE BOOL "Build jerry snapshot command line tool?") set(JERRY_PORT_DEFAULT ON CACHE BOOL "Build default jerry port implementation?") set(JERRY_EXT ON CACHE BOOL "Build jerry-ext?") @@ -58,7 +58,7 @@ set(ENABLE_STRIP ON CACHE BOOL "Enable stripping all symbols from release set(FEATURE_INIT_FINI OFF CACHE BOOL "Enable init/fini arrays?") # Option overrides -if(JERRY_CMDLINE OR JERRY_CMDLINE_MINIMAL OR JERRY_CMDLINE_SNAPSHOT OR UNITTESTS OR DOCTESTS) +if(JERRY_CMDLINE OR JERRY_CMDLINE_TEST OR JERRY_CMDLINE_SNAPSHOT OR UNITTESTS OR DOCTESTS) set(JERRY_PORT_DEFAULT ON) set(JERRY_PORT_DEFAULT_MESSAGE " (FORCED BY CMDLINE OR TESTS)") @@ -103,7 +103,7 @@ message(STATUS "ENABLE_LTO " ${ENABLE_LTO} ${ENABLE_LTO_MESSAGE}) message(STATUS "ENABLE_STATIC_LINK " ${ENABLE_STATIC_LINK} ${ENABLE_STATIC_LINK_MESSAGE}) message(STATUS "ENABLE_STRIP " ${ENABLE_STRIP} ${ENABLE_STRIP_MESSAGE}) message(STATUS "JERRY_CMDLINE " ${JERRY_CMDLINE}) -message(STATUS "JERRY_CMDLINE_MINIMAL " ${JERRY_CMDLINE_MINIMAL}) +message(STATUS "JERRY_CMDLINE_TEST " ${JERRY_CMDLINE_TEST}) message(STATUS "JERRY_CMDLINE_SNAPSHOT " ${JERRY_CMDLINE_SNAPSHOT}) message(STATUS "JERRY_PORT_DEFAULT " ${JERRY_PORT_DEFAULT} ${JERRY_PORT_DEFAULT_MESSAGE}) message(STATUS "JERRY_EXT " ${JERRY_EXT} ${JERRY_EXT_MESSAGE}) @@ -270,7 +270,7 @@ if(JERRY_EXT) endif() # Jerry command line tool -if(JERRY_CMDLINE OR JERRY_CMDLINE_MINIMAL OR JERRY_CMDLINE_SNAPSHOT) +if(JERRY_CMDLINE OR JERRY_CMDLINE_TEST OR JERRY_CMDLINE_SNAPSHOT) add_subdirectory(jerry-main) endif() diff --git a/jerry-libc/jerry-libc-printf.c b/jerry-libc/jerry-libc-printf.c index 7c29978ec..a7868398d 100644 --- a/jerry-libc/jerry-libc-printf.c +++ b/jerry-libc/jerry-libc-printf.c @@ -469,7 +469,7 @@ libc_printf_write_u_o_x_X (FILE *stream, /**< stream pointer */ * * @return number of characters printed */ -int +int __attr_weak___ vfprintf (FILE *stream, /**< stream pointer */ const char *format, /**< format string */ va_list args) /**< arguments */ @@ -719,7 +719,7 @@ vfprintf (FILE *stream, /**< stream pointer */ * * @return number of characters printed */ -int +int __attr_weak___ fprintf (FILE *stream, /**< stream pointer */ const char *format, /**< format string */ ...) /**< parameters' values */ @@ -740,7 +740,7 @@ fprintf (FILE *stream, /**< stream pointer */ * * @return number of characters printed */ -int +int __attr_weak___ printf (const char *format, /**< format string */ ...) /**< parameters' values */ { diff --git a/jerry-libc/jerry-libc.c b/jerry-libc/jerry-libc.c index cde857892..45c44c3e7 100644 --- a/jerry-libc/jerry-libc.c +++ b/jerry-libc/jerry-libc.c @@ -56,7 +56,7 @@ CALL_PRAGMA (GCC optimize ("-fno-tree-loop-distribute-patterns")) * * @return @a s */ -void * __attr_used___ +void * __attr_weak___ __attr_used___ memset (void *s, /**< area to set values in */ int c, /**< value to set */ size_t n) /**< area size */ @@ -77,7 +77,7 @@ memset (void *s, /**< area to set values in */ * <0, if first area's content is lexicographically less, than second area's content; * >0, otherwise */ -int +int __attr_weak___ memcmp (const void *s1, /**< first area */ const void *s2, /**< second area */ size_t n) /**< area size */ @@ -100,7 +100,7 @@ memcmp (const void *s1, /**< first area */ * * @return the dest pointer's value */ -void * __attr_used___ +void * __attr_weak___ __attr_used___ memcpy (void *s1, /**< destination */ const void *s2, /**< source */ size_t n) /**< bytes number */ @@ -139,7 +139,7 @@ memcpy (void *s1, /**< destination */ * * @return the dest pointer's value */ -void * __attr_used___ +void * __attr_weak___ __attr_used___ memmove (void *s1, /**< destination */ const void *s2, /**< source */ size_t n) /**< bytes number */ @@ -182,7 +182,7 @@ CALL_PRAGMA (GCC diagnostic pop) * @return an integer less than, equal to, or greater than zero if s1 is found, respectively, * to be less than, to match, or be greater than s2. */ -int +int __attr_weak___ strcmp (const char *s1, /**< first string */ const char *s2) /**< second string */ { @@ -205,7 +205,7 @@ strcmp (const char *s1, /**< first string */ * @return an integer less than, equal to, or greater than zero if the first n character of s1 is found, respectively, * to be less than, to match, or be greater than the first n character of s2. */ -int +int __attr_weak___ strncmp (const char *s1, /**< first string */ const char *s2, /**< second string */ size_t n) /**< maximum number of characters to compare */ @@ -234,7 +234,7 @@ strncmp (const char *s1, /**< first string */ * * @return a pointer to the destination string dest. */ -char * __attr_used___ +char * __attr_weak___ __attr_used___ strncpy (char *dest, /**< destination string */ const char *src, /**< source string */ size_t n) /**< maximum number of characters to copy */ @@ -258,7 +258,7 @@ strncpy (char *dest, /**< destination string */ * * @return the length. */ -size_t +size_t __attr_weak___ strlen (const char *s) /**< string */ { size_t i = 0; @@ -278,7 +278,7 @@ strlen (const char *s) /**< string */ * * @return integer in range [0; RAND_MAX] */ -int +int __attr_weak___ rand (void) { uint32_t intermediate = libc_random_gen_state[0] ^ (libc_random_gen_state[0] << 11); @@ -297,7 +297,7 @@ rand (void) /** * Initialize pseudo-random number generator with the specified seed value */ -void +void __attr_weak___ srand (unsigned int seed) /**< new seed */ { libc_random_gen_state[0] = (uint32_t) ((seed * 14316555781) @@ -325,7 +325,7 @@ srand (unsigned int seed) /**< new seed */ * * @return the integer value of str. */ -long int +long int __attr_weak___ strtol (const char *nptr, /**< string representation of an integer number */ char **endptr, /**< [out] the address of the first non-number character */ int base) /**< numerical base or radix (MUST be 10) */ diff --git a/jerry-libc/target/posix/jerry-libc-target.c b/jerry-libc/target/posix/jerry-libc-target.c index 019c31b82..d0a628c34 100644 --- a/jerry-libc/target/posix/jerry-libc-target.c +++ b/jerry-libc/target/posix/jerry-libc-target.c @@ -48,7 +48,7 @@ long int syscall_3 (long int syscall_no, long int arg1, long int arg2, long int /** * Exit - cause normal process termination with specified status code */ -void __attr_noreturn___ __attr_used___ +void __attr_weak___ __attr_noreturn___ __attr_used___ exit (int status) /**< status code */ { #ifdef ENABLE_INIT_FINI @@ -71,7 +71,7 @@ exit (int status) /**< status code */ * Abort current process, producing an abnormal program termination. * The function raises the SIGABRT signal. */ -void __attr_noreturn___ __attr_used___ +void __attr_weak___ __attr_noreturn___ __attr_used___ abort (void) { syscall_1 (SYSCALL_NO (close), (long int) stdin); @@ -92,7 +92,7 @@ abort (void) * @return 0 - upon successful completion, * non-zero value - otherwise. */ -int __attr_used___ +int __attr_weak___ __attr_used___ raise (int sig) /**< signal number */ { return (int) syscall_2 (SYSCALL_NO (kill), syscall_0 (SYSCALL_NO (getpid)), sig); @@ -104,7 +104,7 @@ raise (int sig) /**< signal number */ * @return FILE pointer - upon successful completion, * NULL - otherwise. */ -FILE * +FILE * __attr_weak___ fopen (const char *path, /**< file path */ const char *mode) /**< file open mode */ { @@ -193,7 +193,7 @@ fopen (const char *path, /**< file path */ * @return 0 - upon successful completion, * non-zero value - otherwise. */ -int +int __attr_weak___ fclose (FILE *fp) /**< stream pointer */ { syscall_2 (SYSCALL_NO (close), (long int) fp, 0); @@ -206,7 +206,7 @@ fclose (FILE *fp) /**< stream pointer */ * * @return number of elements read */ -size_t +size_t __attr_weak___ fread (void *ptr, /**< address of buffer to read to */ size_t size, /**< size of elements to read */ size_t nmemb, /**< number of elements to read */ @@ -239,7 +239,7 @@ fread (void *ptr, /**< address of buffer to read to */ * * @return number of elements written */ -size_t +size_t __attr_weak___ fwrite (const void *ptr, /**< data to write */ size_t size, /**< size of elements to write */ size_t nmemb, /**< number of elements */ @@ -271,7 +271,7 @@ fwrite (const void *ptr, /**< data to write */ * * @return 0 if success, -1 otherwise */ -int +int __attr_weak___ gettimeofday (void *tp, /**< struct timeval */ void *tzp) /**< struct timezone */ { diff --git a/jerry-main/CMakeLists.txt b/jerry-main/CMakeLists.txt index bffe1f63c..f7dcfc759 100644 --- a/jerry-main/CMakeLists.txt +++ b/jerry-main/CMakeLists.txt @@ -64,9 +64,9 @@ if(JERRY_CMDLINE) target_link_libraries("jerry" jerry-ext jerry-port-default) endif() -if(JERRY_CMDLINE_MINIMAL) - jerry_create_executable("jerry-minimal" "main-unix-minimal.c") - target_link_libraries("jerry-minimal" jerry-port-default-minimal) +if(JERRY_CMDLINE_TEST) + jerry_create_executable("jerry-test" "main-unix-test.c" "benchmarking.c") + target_link_libraries("jerry-test" jerry-port-default-minimal) endif() if(JERRY_CMDLINE_SNAPSHOT) diff --git a/jerry-main/benchmarking.c b/jerry-main/benchmarking.c new file mode 100644 index 000000000..0e8e151a7 --- /dev/null +++ b/jerry-main/benchmarking.c @@ -0,0 +1,63 @@ +/* Copyright JS Foundation and other contributors, http://js.foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This source contains libc overrides for the sake of stable benchmarking. If + * building a binary for the purpose of benchmarking, the object compiled from + * this source is to be injected into the list of objects-to-be-linked before + * the list of libraries-to-be-linked to ensure that the linker picks up these + * implementations. + */ + + #ifdef __GNUC__ +/* + * Note: + * This is nasty and dangerous. However, we only need the timeval structure + * from sys/time.h. Unfortunately, the same header also declares + * gettimeofday, which has different declarations on different platforms + * (e.g., macOS, Linux). So, instead of #ifdef'ing for platforms, we simply + * tweak the header to declare another function. Don't try this at home. + */ +#define gettimeofday __prevent_conflicting_gettimeofday_declarations__ +#include +#undef gettimeofday + +int gettimeofday (struct timeval *, void *); + +/** + * Useless but stable gettimeofday implementation. Returns Epoch. Ensures that + * test cases relying on "now" yield stable results. + */ +int gettimeofday (struct timeval *tv, + void *tz) +{ + (void) tz; + tv->tv_sec = 0; + tv->tv_usec = 0; + return 0; +} /* gettimeofday */ +#endif /* __GNUC__ */ + + +int rand (void); + +/** + * Useless but stable rand implementation. Returns 4. Ensures that test cases + * relying on randomness yield stable results. + */ +int rand (void) +{ + return 4; /* Chosen by fair dice roll. Guaranteed to be random. */ +} /* rand */ diff --git a/jerry-main/main-unix-minimal.c b/jerry-main/main-unix-test.c similarity index 100% rename from jerry-main/main-unix-minimal.c rename to jerry-main/main-unix-test.c diff --git a/tools/build.py b/tools/build.py index 872cea129..06b67b97f 100755 --- a/tools/build.py +++ b/tools/build.py @@ -72,8 +72,6 @@ def get_arguments(): help='Allowed N build jobs at once (default: %(default)s)') parser.add_argument('--jerry-cmdline', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build jerry command line tool (%(choices)s; default: %(default)s)') - parser.add_argument('--jerry-cmdline-minimal', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, - help='build minimal version of the jerry command line tool (%(choices)s; default: %(default)s)') parser.add_argument('--jerry-cmdline-snapshot', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='build snapshot command line tool (%(choices)s; default: %(default)s)') parser.add_argument('--jerry-debugger', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, @@ -118,6 +116,9 @@ def get_arguments(): help='enable VM execution stopping (%(choices)s; default: %(default)s)') devgroup = parser.add_argument_group('developer options') + devgroup.add_argument('--jerry-cmdline-test', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, + help=devhelp('build test version of the jerry command line tool ' + '(%(choices)s; default: %(default)s)')) devgroup.add_argument('--link-map', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable the generation of a link map file for jerry command line tool ' '(%(choices)s; default: %(default)s)')) @@ -151,7 +152,7 @@ def generate_build_options(arguments): build_options.append('-DFEATURE_CPOINTER_32_BIT=%s' % arguments.cpointer_32bit) build_options.append('-DFEATURE_ERROR_MESSAGES=%s' % arguments.error_messages) build_options.append('-DJERRY_CMDLINE=%s' % arguments.jerry_cmdline) - build_options.append('-DJERRY_CMDLINE_MINIMAL=%s' % arguments.jerry_cmdline_minimal) + build_options.append('-DJERRY_CMDLINE_TEST=%s' % arguments.jerry_cmdline_test) build_options.append('-DJERRY_CMDLINE_SNAPSHOT=%s' % arguments.jerry_cmdline_snapshot) build_options.append('-DJERRY_PORT_DEFAULT=%s' % arguments.jerry_port_default) build_options.append('-DJERRY_EXT=%s' % arguments.jerry_ext) diff --git a/tools/run-tests.py b/tools/run-tests.py index 3e474df29..337d1c957 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -122,8 +122,8 @@ JERRY_BUILDOPTIONS = [ ['--jerry-libc=off', '--compile-flag=-m32', '--cpointer-32bit=on', '--system-allocator=on']), Options('buildoption_test-external_context', ['--jerry-libc=off', '--external-context=on']), - Options('buildoption_test-cmdline_minimal', - ['--jerry-cmdline-minimal=on']), + Options('buildoption_test-cmdline_test', + ['--jerry-cmdline-test=on']), Options('buildoption_test-cmdline_snapshot', ['--jerry-cmdline-snapshot=on']), ]