Officially introduce amalgamated builds (#4416)

Remove redundancy between all-in-one and all-in-one-source builds by keeping only the second, and adopt the more established term "amalgamated" build for it. This change includes the following: - Replace `ENABLE_ALL_IN_ONE` and `ENABLE_ALL_IN_ONE_SOURCE` cmake options with `ENABLE_AMALGAM` top-level option. - Replace `--all-in-one` option of `build.py` helper with `--amalgam`. - Merge the `srcmerger.py` and `srcgenerator.py` tool scripts into `amalgam.py` (with improvements). - Update documentation. JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu
2021-01-11 13:07:40 +01:00
parent dcf9252892
commit fdaacde667
16 changed files with 224 additions and 305 deletions
@@ -13,30 +13,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from __future__ import print_function

 import argparse
 import fnmatch
+import json
 import logging
 import os
 import re
-import sys
+import shutil
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+JERRY_CORE = os.path.join(ROOT_DIR, 'jerry-core')
+JERRY_PORT = os.path.join(ROOT_DIR, 'jerry-port', 'default')
+JERRY_MATH = os.path.join(ROOT_DIR, 'jerry-math')


-class SourceMerger(object):
+class Amalgamator(object):
    # pylint: disable=too-many-instance-attributes

    _RE_INCLUDE = re.compile(r'\s*#include ("|<)(.*?)("|>)\n$')

-    def __init__(self, h_files, extra_includes=None, remove_includes=None, add_lineinfo=False):
-        self._log = logging.getLogger('sourcemerger')
+    def __init__(self, h_files, extra_includes=(), remove_includes=(), add_lineinfo=False):
+        self._h_files = h_files
+        self._extra_includes = extra_includes
+        self._remove_includes = remove_includes
+        self._add_lineinfo = add_lineinfo
        self._last_builtin = None
        self._processed = []
        self._output = []
-        self._h_files = h_files
-        self._extra_includes = extra_includes or []
-        self._remove_includes = remove_includes
-        self._add_lineinfo = add_lineinfo
        # The copyright will be loaded from the first input file
        self._copyright = {'lines': [], 'loaded': False}

@@ -44,8 +50,8 @@ class SourceMerger(object):
        # Special case #2: Builtin include header name usage
        if line.strip() == "#include BUILTIN_INC_HEADER_NAME":
            assert self._last_builtin is not None, 'No previous BUILTIN_INC_HEADER_NAME definition'
-            self._log.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: %s',
-                            file_level, self._last_builtin)
+            logging.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: "%s"',
+                          file_level, self._last_builtin)
            self.add_file(self._h_files[self._last_builtin])
            # return from the function as we have processed the included file
            return
@@ -54,8 +60,7 @@ class SourceMerger(object):
        if line.startswith('#define BUILTIN_INC_HEADER_NAME '):
            # the line is in this format: #define BUILTIN_INC_HEADER_NAME "<filename>"
            self._last_builtin = line.split('"', 2)[1]
-            self._log.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: %s',
-                            file_level, self._last_builtin)
+            logging.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: "%s"', file_level, self._last_builtin)

        # the line is not anything special, just push it into the output
        self._output.append(line)
@@ -75,11 +80,11 @@ class SourceMerger(object):

    def add_file(self, filename, file_level=0):
        if os.path.basename(filename) in self._processed:
-            self._log.warning('Tried to to process an already processed file: "%s"', filename)
+            logging.warning('Tried to to process an already processed file: "%s"', filename)
            return

        if not file_level:
-            self._log.debug('Adding file: "%s"', filename)
+            logging.debug('Adding file: "%s"', filename)

        file_level += 1

@@ -111,7 +116,7 @@ class SourceMerger(object):
                    continue

                # check if the line is an '#include' line
-                match = SourceMerger._RE_INCLUDE.match(line)
+                match = self._RE_INCLUDE.match(line)
                if not match:
                    # the line is not a header
                    self._process_non_include(line, file_level)
@@ -125,27 +130,24 @@ class SourceMerger(object):
                name = match.group(2)

                if name in self._remove_includes:
-                    self._log.debug('[%d] Removing include line (%s:%d): %s',
-                                    file_level, filename, line_idx, line.strip())
+                    logging.debug('[%d] Removing include line (%s:%d): %s',
+                                  file_level, filename, line_idx, line.strip())
                    # emit a line info so the line numbering can be tracked correctly
                    self._emit_lineinfo(line_idx + 1, filename)
                    continue

                if name not in self._h_files:
-                    self._log.warning('[%d] Include not found: "%s" in "%s:%d"',
-                                      file_level, name, filename, line_idx)
+                    logging.warning('[%d] Include not found (%s:%d): "%s"', file_level, filename, line_idx, name)
                    self._output.append(line)
                    continue

                if name in self._processed:
-                    self._log.debug('[%d] Already included: "%s"',
-                                    file_level, name)
+                    logging.debug('[%d] Already included: "%s"', file_level, name)
                    # emit a line info so the line numbering can be tracked correctly
                    self._emit_lineinfo(line_idx + 1, filename)
                    continue

-                self._log.debug('[%d] Including: "%s"',
-                                file_level, self._h_files[name])
+                logging.debug('[%d] Including: "%s"', file_level, self._h_files[name])
                self.add_file(self._h_files[name], file_level)

                # mark the continuation of the current file in the output
@@ -198,7 +200,7 @@ def collect_files(base_dir, pattern):
        name = os.path.basename(fname)

        if name in name_mapping:
-            print('Duplicate name detected: "%s" and "%s"' % (name, name_mapping[name]))
+            logging.warning('Duplicate name detected: "%s" and "%s"', fname, name_mapping[name])
            continue

        name_mapping[name] = fname
@@ -206,61 +208,154 @@ def collect_files(base_dir, pattern):
    return name_mapping


-def run_merger(args):
-    h_files = collect_files(args.base_dir, '*.h')
-    c_files = collect_files(args.base_dir, '*.c')
+def amalgamate(base_dir, input_files=(), output_file=None,
+               append_c_files=False, remove_includes=(), extra_includes=(),
+               add_lineinfo=False):
+    """
+    :param input_files: Main input source/header files
+    :param output_file: Output source/header file
+    :param append_c_files: Enable auto inclusion of c files under the base-dir
+    :param add_lineinfo: Enable #line macro insertion into the generated sources
+    """
+    logging.debug('Starting merge with args: %s', json.dumps(locals(), indent=4, sort_keys=True))

-    for name in args.remove_include:
+    h_files = collect_files(base_dir, '*.h')
+    c_files = collect_files(base_dir, '*.c')
+
+    for name in remove_includes:
        c_files.pop(name, '')
        h_files.pop(name, '')

-    merger = SourceMerger(h_files, args.push_include, args.remove_include, args.add_lineinfo)
-    for input_file in args.input_files:
-        merger.add_file(input_file)
+    amalgam = Amalgamator(h_files, extra_includes, remove_includes, add_lineinfo)
+    for input_file in input_files:
+        amalgam.add_file(input_file)

-    if args.append_c_files:
+    if append_c_files:
        # if the input file is in the C files list it should be removed to avoid
        # double inclusion of the file
-        for input_file in args.input_files:
+        for input_file in input_files:
            input_name = os.path.basename(input_file)
            c_files.pop(input_name, '')

        # Add the C files in reverse order to make sure that builtins are
        # not at the beginning.
        for fname in sorted(c_files.values(), reverse=True):
-            merger.add_file(fname)
+            amalgam.add_file(fname)

-    with open(args.output_file, 'w') as output:
-        merger.write_output(output)
+    with open(output_file, 'w') as output:
+        amalgam.write_output(output)


+def amalgamate_jerry_core(output_dir):
+    amalgamate(
+        base_dir=JERRY_CORE,
+        input_files=[
+            os.path.join(JERRY_CORE, 'api', 'jerry.c'),
+            # Add the global built-in by default to include some common items
+            # to avoid problems with common built-in headers
+            os.path.join(JERRY_CORE, 'ecma', 'builtin-objects', 'ecma-builtins.c'),
+        ],
+        output_file=os.path.join(output_dir, 'jerryscript.c'),
+        append_c_files=True,
+        remove_includes=[
+            'jerryscript.h',
+            'jerryscript-port.h',
+            'jerryscript-compiler.h',
+            'jerryscript-core.h',
+            'jerryscript-debugger.h',
+            'jerryscript-debugger-transport.h',
+            'jerryscript-port.h',
+            'jerryscript-snapshot.h',
+            'config.h',
+        ],
+        extra_includes=['jerryscript.h'],
+    )
+
+    amalgamate(
+        base_dir=JERRY_CORE,
+        input_files=[
+            os.path.join(JERRY_CORE, 'include', 'jerryscript.h'),
+            os.path.join(JERRY_CORE, 'include', 'jerryscript-debugger-transport.h'),
+        ],
+        output_file=os.path.join(output_dir, 'jerryscript.h'),
+        remove_includes=['config.h'],
+        extra_includes=['jerryscript-config.h'],
+    )
+
+    shutil.copyfile(os.path.join(JERRY_CORE, 'config.h'),
+                    os.path.join(output_dir, 'jerryscript-config.h'))
+
+
+def amalgamate_jerry_port_default(output_dir):
+    amalgamate(
+        base_dir=JERRY_PORT,
+        output_file=os.path.join(output_dir, 'jerryscript-port-default.c'),
+        append_c_files=True,
+        remove_includes=[
+            'jerryscript-port.h',
+            'jerryscript-port-default.h',
+            'jerryscript-debugger.h',
+        ],
+        extra_includes=[
+            'jerryscript.h',
+            'jerryscript-port-default.h',
+        ],
+    )
+
+    amalgamate(
+        base_dir=JERRY_PORT,
+        input_files=[os.path.join(JERRY_PORT, 'include', 'jerryscript-port-default.h')],
+        output_file=os.path.join(output_dir, 'jerryscript-port-default.h'),
+        remove_includes=[
+            'jerryscript-port.h',
+            'jerryscript.h',
+        ],
+        extra_includes=['jerryscript.h'],
+    )
+
+
+def amalgamate_jerry_math(output_dir):
+    amalgamate(
+        base_dir=JERRY_MATH,
+        output_file=os.path.join(output_dir, 'jerryscript-math.c'),
+        append_c_files=True,
+    )
+
+    shutil.copyfile(os.path.join(JERRY_MATH, 'include', 'math.h'),
+                    os.path.join(output_dir, 'math.h'))
+
 def main():
-    parser = argparse.ArgumentParser(description='Merge source/header files.')
-    parser.add_argument('--base-dir', metavar='DIR', type=str, dest='base_dir',
-                        help='', default=os.path.curdir)
-    parser.add_argument('--input', metavar='FILES', type=str, action='append', dest='input_files',
-                        help='Main input source/header files', default=[])
-    parser.add_argument('--output', metavar='FILE', type=str, dest='output_file',
-                        help='Output source/header file')
-    parser.add_argument('--append-c-files', dest='append_c_files', default=False,
-                        action='store_true', help='Enable auto inclusion of c files under the base-dir')
-    parser.add_argument('--remove-include', action='append', default=[])
-    parser.add_argument('--push-include', action='append', default=[])
-    parser.add_argument('--add-lineinfo', action='store_true', default=False,
-                        help='Enable #line macro insertion into the generated sources')
-    parser.add_argument('--verbose', '-v', action='store_true', default=False)
+    parser = argparse.ArgumentParser(description='Generate amalgamated sources.')
+    parser.add_argument('--jerry-core', action='store_true',
+                        help='amalgamate jerry-core files')
+    parser.add_argument('--jerry-port-default', action='store_true',
+                        help='amalgamate jerry-port-default files')
+    parser.add_argument('--jerry-math', action='store_true',
+                        help='amalgamate jerry-math files')
+    parser.add_argument('--output-dir', metavar='DIR', default='amalgam',
+                        help='output dir (default: %(default)s)')
+    parser.add_argument('--verbose', '-v', action='store_true',
+                        help='increase verbosity')

    args = parser.parse_args()

-    log_level = logging.WARNING
    if args.verbose:
-        log_level = logging.DEBUG
+        logging.basicConfig(level=logging.DEBUG)

-    logging.basicConfig(level=log_level)
-    logging.debug('Starting merge with args: %s', str(sys.argv))
+    try:
+        os.makedirs(args.output_dir)
+    except os.error:
+        pass

-    run_merger(args)
+    if args.jerry_core:
+        amalgamate_jerry_core(args.output_dir)
+
+    if args.jerry_port_default:
+        amalgamate_jerry_port_default(args.output_dir)
+
+    if args.jerry_math:
+        amalgamate_jerry_math(args.output_dir)


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -76,6 +76,8 @@ def get_arguments():
                          help='add custom library to be linked')
    buildgrp.add_argument('--linker-flag', metavar='OPT', action='append', default=[],
                          help='add custom linker flag')
+    buildgrp.add_argument('--amalgam', metavar='X', choices=['ON', 'OFF'], type=str.upper,
+                          help='enable amalgamated build (%(choices)s)')
    buildgrp.add_argument('--lto', metavar='X', choices=['ON', 'OFF'], type=str.upper,
                          help='enable link-time optimizations (%(choices)s)')
    buildgrp.add_argument('--shared-libs', metavar='X', choices=['ON', 'OFF'], type=str.upper,
@@ -108,8 +110,6 @@ def get_arguments():
                         help=devhelp('build unittests (%(choices)s)'))

    coregrp = parser.add_argument_group('jerry-core options')
-    coregrp.add_argument('--all-in-one', metavar='X', choices=['ON', 'OFF'], type=str.upper,
-                         help='all-in-one build (%(choices)s)')
    coregrp.add_argument('--cpointer-32bit', metavar='X', choices=['ON', 'OFF'], type=str.upper,
                         help='enable 32 bit compressed pointers (%(choices)s)')
    coregrp.add_argument('--error-messages', metavar='X', choices=['ON', 'OFF'], type=str.upper,
@@ -180,6 +180,7 @@ def generate_build_options(arguments):
    build_options_append('EXTERNAL_COMPILE_FLAGS', ' '.join(arguments.compile_flag))
    build_options_append('EXTERNAL_LINK_LIBS', ' '.join(arguments.link_lib))
    build_options_append('EXTERNAL_LINKER_FLAGS', ' '.join(arguments.linker_flag))
+    build_options_append('ENABLE_AMALGAM', arguments.amalgam)
    build_options_append('ENABLE_LTO', arguments.lto)
    build_options_append('BUILD_SHARED_LIBS', arguments.shared_libs)
    build_options_append('ENABLE_STRIP', arguments.strip)
@@ -198,7 +199,6 @@ def generate_build_options(arguments):
    build_options_append('UNITTESTS', arguments.unittests)

    # jerry-core options
-    build_options_append('ENABLE_ALL_IN_ONE', arguments.all_in_one)
    build_options_append('JERRY_CPOINTER_32_BIT', arguments.cpointer_32bit)
    build_options_append('JERRY_ERROR_MESSAGES', arguments.error_messages)
    build_options_append('JERRY_EXTERNAL_CONTEXT', arguments.external_context)
@@ -129,8 +129,8 @@ JERRY_BUILDOPTIONS = [
            ['--error-messages=on']),
    Options('buildoption_test-logging',
            ['--logging=on']),
-    Options('buildoption_test-all_in_one',
-            ['--all-in-one=on']),
+    Options('buildoption_test-amalgam',
+            ['--amalgam=on']),
    Options('buildoption_test-valgrind',
            ['--valgrind=on']),
    Options('buildoption_test-mem_stats',
@@ -163,8 +163,6 @@ JERRY_BUILDOPTIONS = [
            OPTIONS_STACK_LIMIT),
    Options('buildoption_test-gc-mark_limit',
            OPTIONS_GC_MARK_LIMIT),
-    Options('buildoption_test-single-source',
-            ['--cmake-param=-DENABLE_ALL_IN_ONE_SOURCE=ON']),
    Options('buildoption_test-jerry-debugger',
            ['--jerry-debugger=on']),
    Options('buildoption_test-module-off',
@@ -1,154 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright JS Foundation and other contributors, http://js.foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import print_function
-
-import argparse
-import logging
-import os
-import subprocess
-import shutil
-
-
-TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
-ROOT_DIR = os.path.dirname(TOOLS_DIR)
-SRCMERGER = os.path.join(TOOLS_DIR, 'srcmerger.py')
-JERRY_CORE = os.path.join(ROOT_DIR, 'jerry-core')
-JERRY_PORT = os.path.join(ROOT_DIR, 'jerry-port', 'default')
-JERRY_MATH = os.path.join(ROOT_DIR, 'jerry-math')
-
-
-def run_commands(*cmds, **kwargs):
-    log = logging.getLogger('sourcegenerator')
-    verbose = kwargs.get('verbose', False)
-
-    for cmd in cmds:
-        if verbose:
-            cmd.append('--verbose')
-        log.debug('Run command: %s', cmd)
-        subprocess.call(cmd)
-
-
-def generate_jerry_core(output_dir, verbose=False):
-    cmd_jerry_c_gen = [
-        'python', SRCMERGER,
-        '--base-dir', JERRY_CORE,
-        '--input={}/api/jerry.c'.format(JERRY_CORE),
-        '--output={}/jerryscript.c'.format(output_dir),
-        '--append-c-files',
-        # Add the global built-in by default to include some common items
-        # to avoid problems with common built-in headers
-        '--input={}/ecma/builtin-objects/ecma-builtins.c'.format(JERRY_CORE),
-        '--remove-include=jerryscript.h',
-        '--remove-include=jerryscript-port.h',
-        '--remove-include=jerryscript-compiler.h',
-        '--remove-include=jerryscript-core.h',
-        '--remove-include=jerryscript-debugger.h',
-        '--remove-include=jerryscript-debugger-transport.h',
-        '--remove-include=jerryscript-port.h',
-        '--remove-include=jerryscript-snapshot.h',
-        '--remove-include=config.h',
-        '--push-include=jerryscript.h',
-    ]
-
-    cmd_jerry_h_gen = [
-        'python', SRCMERGER,
-        '--base-dir', JERRY_CORE,
-        '--input={}/include/jerryscript.h'.format(JERRY_CORE),
-        '--input={}/include/jerryscript-debugger-transport.h'.format(JERRY_CORE),
-        '--output={}/jerryscript.h'.format(output_dir),
-        '--remove-include=config.h',
-        '--push-include=jerryscript-config.h',
-    ]
-
-    run_commands(cmd_jerry_c_gen, cmd_jerry_h_gen, verbose=verbose)
-
-    shutil.copyfile('{}/config.h'.format(JERRY_CORE),
-                    '{}/jerryscript-config.h'.format(output_dir))
-
-
-def generate_jerry_port_default(output_dir, verbose=False):
-    cmd_port_c_gen = [
-        'python', SRCMERGER,
-        '--base-dir', JERRY_PORT,
-        '--output={}/jerryscript-port-default.c'.format(output_dir),
-        '--append-c-files',
-        '--remove-include=jerryscript-port.h',
-        '--remove-include=jerryscript-port-default.h',
-        '--remove-include=jerryscript-debugger.h',
-        '--push-include=jerryscript.h',
-        '--push-include=jerryscript-port-default.h',
-    ]
-
-    cmd_port_h_gen = [
-        'python', SRCMERGER,
-        '--base-dir', JERRY_PORT,
-        '--input={}/include/jerryscript-port-default.h'.format(JERRY_PORT),
-        '--output={}/jerryscript-port-default.h'.format(output_dir),
-        '--remove-include=jerryscript-port.h',
-        '--remove-include=jerryscript.h',
-        '--push-include=jerryscript.h',
-    ]
-
-    run_commands(cmd_port_c_gen, cmd_port_h_gen, verbose=verbose)
-
-
-def generate_jerry_math(output_dir, verbose=False):
-    cmd_math_c_gen = [
-        'python', SRCMERGER,
-        '--base-dir', JERRY_MATH,
-        '--output={}/jerryscript-math.c'.format(output_dir),
-        '--append-c-files',
-    ]
-
-    run_commands(cmd_math_c_gen, verbose=verbose)
-
-    shutil.copyfile('{}/include/math.h'.format(JERRY_MATH),
-                    '{}/math.h'.format(output_dir))
-
-def main():
-    parser = argparse.ArgumentParser(description='Generate single sources.')
-    parser.add_argument('--jerry-core', action='store_true', dest='jerry_core',
-                        help='Generate jerry-core files', default=False)
-    parser.add_argument('--jerry-port-default', action='store_true', dest='jerry_port_default',
-                        help='Generate jerry-port-default files', default=False)
-    parser.add_argument('--jerry-math', action='store_true', dest='jerry_math',
-                        help='Generate jerry-math files', default=False)
-    parser.add_argument('--output-dir', metavar='DIR', type=str, dest='output_dir',
-                        default='gen_src', help='Output dir')
-    parser.add_argument('--verbose', '-v', action='store_true', default=False)
-
-    args = parser.parse_args()
-
-    if args.verbose:
-        logging.basicConfig(level=logging.DEBUG)
-
-    try:
-        os.makedirs(args.output_dir)
-    except os.error:
-        pass
-
-    if args.jerry_core:
-        generate_jerry_core(args.output_dir, args.verbose)
-
-    if args.jerry_port_default:
-        generate_jerry_port_default(args.output_dir, args.verbose)
-
-    if args.jerry_math:
-        generate_jerry_math(args.output_dir, args.verbose)
-
-
-if __name__ == '__main__':
-    main()