Document and improve the single-source generation (#2986)

Changes done:
* Added usage/configuration info for this mode.
* Created `tools/srcgenerator.py` to allow source/header generation
  without using CMake.
* Adapted CMake to use the `srcgenerator.py` script.
* Added jerry-libm single-source build.
* Improved the `srcmerger.py` script to correctly handle the line numbering.

JerryScript-DCO-1.0-Signed-off-by: Peter Gal pgal.u-szeged@partner.samsung.com

CMakeLists.txt

@@ -231,6 +231,9 @@ if(DEFINED EXTERNAL_LINKER_FLAGS)
   jerry_add_link_flags(${EXTERNAL_LINKER_FLAGS})
 endif()
 
+# Used for placeholder to attach single source build targets
+add_custom_target(generate-single-source)
+
 # Jerry's libm
 if(JERRY_LIBM)
   add_subdirectory(jerry-libm)

docs/01.CONFIGURATION.md

@@ -250,3 +250,40 @@ This option is disabled by default.
 | C:      | `-DJERRY_MEM_GC_BEFORE_EACH_ALLOC=0/1`       |
 | CMake:  | `-DJERRY_MEM_GC_BEFORE_EACH_ALLOC=ON/OFF`    |
 | Python: | `--mem-stress-test=ON/OFF`                   |
+
+
+# Single source build mode
+
+There is a special mode to use/"build" JerryScript. That is generating a single C file which can be
+included into projects quickly. To achive this the following command can be executed to create
+a set of files into the `gen_src` directory (Note that the command is executed in the jerryscript root directory
+but can be adapted to run outside of the project root dir):
+
+```sh
+$ python tools/srcgenerator.py --output-dir gen_src --jerry-core --jerry-port-default --jerry-libm
+```
+
+The command creates the following files in the `gen_src` dir:
+
+* `jerryscript.c`
+* `jerryscript.h`
+* `jerryscript-config.h`
+* `jerryscript-port-default.c`
+* `jerryscript-port-default.h`
+* `jerryscript-libm.c`
+* `math.h`
+
+**Important**: the `jerryscript-config.h` contains the configurations mentioned above and
+should be adapted to the required use-case. See the file contents for more details and for the
+default configuration. (Note: This config file is created from the the `jerry-core/config.h` file.)
+
+These files can be directly compiled with an application using the JerryScript API.
+For example with the following command:
+
+```sh
+$ gcc -Wall -o demo_app demo_app.c gen_src/jerryscript.c gen_src/jerryscript-port-default.c jerryscript-libm.c -Igen_src/ 
+```
+
+Please note that the headers must be available on the include path.
+
+In addition there is a `-DENABLE_ALL_IN_ONE_SOURCE=ON` CMake option to use this kind of sources during the build.

jerry-core/CMakeLists.txt

@@ -160,48 +160,28 @@ endif()
 #   * jerryscript.h
 #   * jerryscript-config.h
 if(ENABLE_ALL_IN_ONE_SOURCE)
-  # Create a default configuration
-  set(JERRYSCRIPT_CONFIG_H "${CMAKE_BINARY_DIR}/jerryscript-config.h")
-  set(JERRYSCRIPT_SOURCE_CONFIG_H "${CMAKE_CURRENT_SOURCE_DIR}/config.h")
-  add_custom_command(OUTPUT ${JERRYSCRIPT_CONFIG_H}
-                     COMMAND ${CMAKE_COMMAND} -E copy ${JERRYSCRIPT_SOURCE_CONFIG_H} ${JERRYSCRIPT_CONFIG_H}
-                     DEPENDS ${JERRYSCRIPT_SOURCE_CONFIG_H})
 
-  # Create single C file
+  # Create single C/H file
   file(GLOB HEADER_CORE_FILES *.h)
-  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/jerryscript.c")
+
-  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/jerryscript.h")
+  # Generated files
-  add_custom_command(OUTPUT ${ALL_IN_FILE}
+  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript.c")
-                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/jerryscript.h")
-                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
+  set(JERRYSCRIPT_CONFIG_H "${CMAKE_BINARY_DIR}/src/jerryscript-config.h")
-                             --input ${CMAKE_CURRENT_SOURCE_DIR}/api/jerry.c
+
-                             --output ${ALL_IN_FILE}
+  add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}
-                             --append-c-files
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
-                             --remove-include jerryscript.h
+                             --jerry-core
-                             --remove-include jerryscript-port.h
+                             --output-dir ${CMAKE_BINARY_DIR}/src
-                             --remove-include jerryscript-compiler.h
+                     DEPENDS ${SOURCE_CORE_FILES}
-                             --remove-include jerryscript-core.h
+                             ${HEADER_CORE_FILES}
-                             --remove-include jerryscript-debugger.h
+                             ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
-                             --remove-include jerryscript-debugger-transport.h
+                             ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
-                             --remove-include jerryscript-port.h
-                             --remove-include jerryscript-snapshot.h
-                             --remove-include config.h
-                             --push-include jerryscript.h
-                    DEPENDS ${SOURCE_CORE_FILES} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}
-  )
-  add_custom_command(OUTPUT ${ALL_IN_FILE_H}
-                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
-                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
-                             --input ${CMAKE_CURRENT_SOURCE_DIR}/include/jerryscript.h
-                             --output ${ALL_IN_FILE_H}
-                             --remove-include config.h
-                             --push-include jerryscript-config.h
-                     DEPENDS ${HEADER_CORE_FILES} ${JERRYSCRIPT_CONFIG_H}
   )
   add_custom_target(generate-single-source-jerry DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H})
-  add_custom_target(generate-single-source DEPENDS generate-single-source-jerry)
+  add_dependencies(generate-single-source generate-single-source-jerry)
 
-  set(SOURCE_CORE_FILES ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+  set(SOURCE_CORE_FILES ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H})
 endif()
 
 # Third-party

jerry-libm/CMakeLists.txt

@@ -30,6 +30,30 @@ set(INCLUDE_LIBM "${CMAKE_CURRENT_SOURCE_DIR}/include")
 # Source directories
 file(GLOB SOURCE_LIBM *.c)
 
+# "Single" JerryScript libm source/header build.
+#  The process will create the following files:
+#   * jerryscript-libm.c
+#   * math.h
+if(ENABLE_ALL_IN_ONE_SOURCE)
+  file(GLOB HEADER_LIBM *.h)
+  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript-libm.c")
+  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/math.h")
+
+  add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
+                             --jerry-libm
+                             --output-dir ${CMAKE_BINARY_DIR}/src
+                     DEPENDS ${SOURCE_LIBM}
+                             ${HEADER_LIBM}
+                             ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
+                             ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+  )
+  add_custom_target(generate-single-source-libm DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+  add_dependencies(generate-single-source generate-single-source-libm)
+
+  set(SOURCE_LIBM ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+endif()
+
 add_library(${JERRY_LIBM_NAME} ${SOURCE_LIBM})
 set_property(TARGET ${JERRY_LIBM_NAME}
              PROPERTY COMPILE_FLAGS "${COMPILE_FLAGS_LIBM}")

jerry-port/default/CMakeLists.txt

@@ -28,30 +28,17 @@ file(GLOB SOURCE_PORT_DEFAULT *.c)
 #   * jerryscript-port-default.h
 if(ENABLE_ALL_IN_ONE_SOURCE)
   file(GLOB HEADER_PORT_DEFAULT *.h)
-  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/jerryscript-port-default.c")
+  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/src/jerryscript-port-default.c")
-  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/jerryscript-port-default.h")
+  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/src/jerryscript-port-default.h")
-  add_custom_command(OUTPUT ${ALL_IN_FILE}
-                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
-                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
-                             --output ${ALL_IN_FILE}
-                             --append-c-files
-                             --remove-include jerryscript-port.h
-                             --remove-include jerryscript-port-default.h
-                             --remove-include jerryscript-debugger.h
-                             --push-include jerryscript.h
-                             --push-include jerryscript-port-default.h
-                     DEPENDS ${SOURCE_PORT_DEFAULT} ${ALL_IN_FILE_H}
-  )
 
-  add_custom_command(OUTPUT ${ALL_IN_FILE_H}
+  add_custom_command(OUTPUT ${ALL_IN_FILE} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}
-                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
-                               --base-dir ${CMAKE_CURRENT_SOURCE_DIR}/
+                             --jerry-port-default
-                               --input ${CMAKE_CURRENT_SOURCE_DIR}/include/jerryscript-port-default.h
+                             --output-dir ${CMAKE_BINARY_DIR}/src
-                               --output ${ALL_IN_FILE_H}
+                     DEPENDS ${SOURCE_PORT_DEFAULT}
-                               --remove-include jerryscript-port.h
+                             ${HEADER_PORT_DEFAULT}
-                               --remove-include jerryscript.h
+                             ${CMAKE_SOURCE_DIR}/tools/srcgenerator.py
-                               --push-include jerryscript.h
+                             ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
-                    DEPENDS ${HEADER_PORT_DEFAULT}
   )
   add_custom_target(generate-single-source-port DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H})
   add_dependencies(generate-single-source generate-single-source-port)

tools/srcgenerator.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(TOOLS_DIR)
+SRCMERGER = os.path.join(TOOLS_DIR, 'srcmerger.py')
+JERRY_CORE = os.path.join(ROOT_DIR, 'jerry-core')
+JERRY_PORT = os.path.join(ROOT_DIR, 'jerry-port', 'default')
+JERRY_LIBM = os.path.join(ROOT_DIR, 'jerry-libm')
+
+
+def run_commands(*cmds, **kwargs):
+    log = logging.getLogger('sourcegenerator')
+    verbose = kwargs.get('verbose', False)
+
+    for cmd in cmds:
+        if verbose:
+            cmd.append('--verbose')
+        log.debug('Run command: %s', cmd)
+        subprocess.call(cmd)
+
+
+def generate_jerry_core(output_dir, verbose=False):
+    cmd_jerry_c_gen = [
+        'python', SRCMERGER,
+        '--base-dir', JERRY_CORE,
+        '--input={}/api/jerry.c'.format(JERRY_CORE),
+        '--output={}/jerryscript.c'.format(output_dir),
+        '--append-c-files',
+        '--remove-include=jerryscript.h',
+        '--remove-include=jerryscript-port.h',
+        '--remove-include=jerryscript-compiler.h',
+        '--remove-include=jerryscript-core.h',
+        '--remove-include=jerryscript-debugger.h',
+        '--remove-include=jerryscript-debugger-transport.h',
+        '--remove-include=jerryscript-port.h',
+        '--remove-include=jerryscript-snapshot.h'
+        '--remove-include=config.h',
+        '--push-include=jerryscript.h',
+    ]
+
+    cmd_jerry_h_gen = [
+        'python', SRCMERGER,
+        '--base-dir', JERRY_CORE,
+        '--input={}/include/jerryscript.h'.format(JERRY_CORE),
+        '--output={}/jerryscript.h'.format(output_dir),
+        '--remove-include=config.h',
+        '--push-include=jerryscript-config.h',
+    ]
+
+    run_commands(cmd_jerry_c_gen, cmd_jerry_h_gen, verbose=verbose)
+
+    shutil.copyfile('{}/config.h'.format(JERRY_CORE),
+                    '{}/jerryscript-config.h'.format(output_dir))
+
+
+def generate_jerry_port_default(output_dir, verbose=False):
+    cmd_port_c_gen = [
+        'python', SRCMERGER,
+        '--base-dir', JERRY_PORT,
+        '--output={}/jerryscript-port-default.c'.format(output_dir),
+        '--append-c-files',
+        '--remove-include=jerryscript-port.h',
+        '--remove-include=jerryscript-port-default.h',
+        '--remove-include=jerryscript-debugger.h',
+        '--push-include=jerryscript.h',
+        '--push-include=jerryscript-port-default.h',
+    ]
+
+    cmd_port_h_gen = [
+        'python', SRCMERGER,
+        '--base-dir', JERRY_PORT,
+        '--input={}/include/jerryscript-port-default.h'.format(JERRY_PORT),
+        '--output={}/jerryscript-port-default.h'.format(output_dir),
+        '--remove-include=jerryscript-port.h',
+        '--remove-include=jerryscript.h',
+        '--push-include=jerryscript.h',
+    ]
+
+    run_commands(cmd_port_c_gen, cmd_port_h_gen, verbose=verbose)
+
+
+def generate_jerry_libm(output_dir, verbose=False):
+    cmd_libm_c_gen = [
+        'python', SRCMERGER,
+        '--base-dir', JERRY_LIBM,
+        '--output={}/jerryscript-libm.c'.format(output_dir),
+        '--append-c-files',
+    ]
+
+    run_commands(cmd_libm_c_gen, verbose=verbose)
+
+    shutil.copyfile('{}/include/math.h'.format(JERRY_LIBM),
+                    '{}/math.h'.format(output_dir))
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate single sources.')
+    parser.add_argument('--jerry-core', action='store_true', dest='jerry_core',
+                        help='Generate jerry-core files', default=False)
+    parser.add_argument('--jerry-port-default', action='store_true', dest='jerry_port_default',
+                        help='Generate jerry-port-default files', default=False)
+    parser.add_argument('--jerry-libm', action='store_true', dest='jerry_libm',
+                        help='Generate jerry-libm files', default=False)
+    parser.add_argument('--output-dir', metavar='DIR', type=str, dest='output_dir',
+                        default='gen_src', help='Output dir')
+    parser.add_argument('--verbose', '-v', action='store_true', default=False)
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    try:
+        os.makedirs(args.output_dir)
+    except os.error:
+        pass
+
+    if args.jerry_core:
+        generate_jerry_core(args.output_dir, args.verbose)
+
+    if args.jerry_port_default:
+        generate_jerry_port_default(args.output_dir, args.verbose)
+
+    if args.jerry_libm:
+        generate_jerry_libm(args.output_dir, args.verbose)
+
+
+if __name__ == '__main__':
+    main()

tools/srcmerger.py

@@ -59,6 +59,15 @@ class SourceMerger(object):
         # the line is not anything special, just push it into the output
         self._output.append(line)
 
+    def _emit_lineinfo(self, line_number, filename):
+        line_info = '#line %d "%s"\n' % (line_number, filename)
+
+        if self._output and self._output[-1].startswith('#line'):
+            # Avoid emitting multiple line infos in sequence, just overwrite the last one
+            self._output[-1] = line_info
+        else:
+            self._output.append(line_info)
+
     def add_file(self, filename, file_level=0):
         if os.path.basename(filename) in self._processed:
             self._log.warning('Tried to to process an already processed file: "%s"', filename)
@@ -67,7 +76,7 @@ class SourceMerger(object):
         file_level += 1
 
         # mark the start of the new file in the output
-        self._output.append('#line 1 "%s"\n' % (filename))
+        self._emit_lineinfo(1, filename)
 
         line_idx = 0
         with open(filename, 'r') as input_file:
@@ -88,6 +97,8 @@ class SourceMerger(object):
                     if line.strip().endswith('*/'):
                         in_copyright = False
                         self._copyright['loaded'] = True
+                        # emit a line info so the line numbering can be tracked correctly
+                        self._emit_lineinfo(line_idx + 1, filename)
 
                     continue
 
@@ -108,6 +119,8 @@ class SourceMerger(object):
                 if name in self._remove_includes:
                     self._log.debug('[%d] Removing include line (%s:%d): %s',
                                     file_level, filename, line_idx, line.strip())
+                    # emit a line info so the line numbering can be tracked correctly
+                    self._emit_lineinfo(line_idx + 1, filename)
                     continue
 
                 if name not in self._h_files:
@@ -119,6 +132,8 @@ class SourceMerger(object):
                 if name in self._processed:
                     self._log.debug('[%d] Already included: "%s"',
                                     file_level, name)
+                    # emit a line info so the line numbering can be tracked correctly
+                    self._emit_lineinfo(line_idx + 1, filename)
                     continue
 
                 self._log.debug('[%d] Including: "%s"',
@@ -126,7 +141,7 @@ class SourceMerger(object):
                 self.add_file(self._h_files[name])
 
                 # mark the continuation of the current file in the output
-                self._output.append('#line %d "%s"\n' % (line_idx + 1, filename))
+                self._emit_lineinfo(line_idx + 1, filename)
 
                 if not name.endswith('.inc.h'):
                     # if the included file is not a "*.inc.h" file mark it as processed