Refactor js2c.py and eliminate related pylint warnings (#1728)

JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com

tools/c_source_helper.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 LICENSE = """/* Copyright JS Foundation and other contributors, http://js.foundation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,7 +31,7 @@ LICENSE = """/* Copyright JS Foundation and other contributors, http://js.founda
  */"""
 
 
-class Source(object):
+class UniCodeSource(object):
     def __init__(self, filepath):
         self.__filepath = filepath
         self.__header = [LICENSE, ""]
@@ -38,7 +39,7 @@ class Source(object):
 
     def complete_header(self, completion):
         self.__header.append(completion)
-        self.__header.append("") # for an extra empty line
+        self.__header.append("")  # for an extra empty line
 
     def add_table(self, table, table_name, table_type, table_descr):
         self.__data.append(table_descr)
@@ -46,30 +47,33 @@ class Source(object):
         self.__data.append("{")
         self.__data.append(format_code(table, 1))
         self.__data.append("};")
-        self.__data.append("") # for an extra empty line
+        self.__data.append("")  # for an extra empty line
 
     def generate(self):
-        with open(self.__filepath, 'w') as genereted_source:
-            genereted_source.write("\n".join(self.__header))
-            genereted_source.write("\n".join(self.__data))
+        with open(self.__filepath, 'w') as generated_source:
+            generated_source.write("\n".join(self.__header))
+            generated_source.write("\n".join(self.__data))
 
 
 def regroup(list_to_group, num):
     return [list_to_group[i:i+num] for i in range(0, len(list_to_group), num)]
 
 
-def hex_format(char):
+def hex_format(char, digit_number):
     if isinstance(char, str):
         char = ord(char)
 
-    return "0x{:04x}".format(char)
+    return ("0x{:0%sx}" % digit_number).format(char)
 
 
-def format_code(code, indent):
+def format_code(code, indent, digit_number=4):
     lines = []
+
+    nums_per_line = 10
+    width = nums_per_line * (digit_number + 4)
     # convert all characters to hex format
-    converted_code = [hex_format(char) for char in code]
+    converted_code = [hex_format(char, digit_number) for char in code]
     # 10 hex number per line
-    for line in regroup(", ".join(converted_code), 10 * 8):
+    for line in regroup(", ".join(converted_code), width):
         lines.append(('  ' * indent) + line.strip())
     return "\n".join(lines)

tools/js2c.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#  This file converts ./js/*.js to a C-array in ./source/jerry-targetjs.h file
+
+import argparse
+import glob
+import os
+import re
+import c_source_helper
+
+HEADER = '''#ifndef JERRY_TARGETJS_H
+#define JERRY_TARGETJS_H
+'''
+
+FOOTER = '''
+#endif
+'''
+
+NATIVE_STRUCT = '''
+struct js_source_all {
+  const char* name;
+  const char* source;
+  const int length;
+};
+
+#define DECLARE_JS_CODES \\
+struct js_source_all js_codes[] = \\
+{ \\'''
+
+
+def extract_name(path):
+    special_chars = re.compile(r'[-\\?\'".]')
+    return special_chars.sub('_', os.path.splitext(os.path.basename(path))[0])
+
+
+def reduce_code(code):
+    code = re.sub(r"/\*.*?\*/", "", code, flags=re.DOTALL)  # remove all occurance streamed comments
+    code = re.sub(r"//.*?\n", "", code)  # remove all occurance singleline comments
+    code = re.sub('\n+', '\n', re.sub('\n +', '\n', code))  # remove white spaces
+    return code
+
+
+def js_to_native_code(path, name, build_type):
+    with open(path, 'r') as js_source:
+        code = js_source.read()
+
+    if build_type != 'debug':
+        code = reduce_code(code)
+
+    data = c_source_helper.format_code(code, 1, 2)
+
+    native_code = """const static char {0}_n[] = "{0}";
+const static char {0}_s[] =
+{{
+{1}
+}};
+const static int {0}_l = {2};
+""".format(name, data, len(code))
+
+    return native_code
+
+
+def main():
+    parser = argparse.ArgumentParser(description="js2c")
+    parser.add_argument('--build-type', help='build type', default='release', choices=['release', 'debug'])
+    parser.add_argument('--ignore', help='files to ignore', dest='ignore_files', default=[], action='append')
+    parser.add_argument('--no-main',
+                        help="don't require a 'main.js' file",
+                        dest='main',
+                        action='store_false',
+                        default=True)
+    parser.add_argument('--js-source',
+                        dest='js_source_path',
+                        default='./js',
+                        help='Source directory of JavaScript files" (default: %(default)s)')
+    parser.add_argument('--dest',
+                        dest='output_path',
+                        default='./source',
+                        help="Destination directory of 'jerry-targetjs.h' (default: %(default)s)")
+
+    script_args = parser.parse_args()
+
+    gen_line = "/* This file is generated by %s. Please do not modify. */" % os.path.basename(__file__)
+
+    gen_output = [c_source_helper.LICENSE, "", gen_line, "", HEADER]
+    gen_structs = [NATIVE_STRUCT]
+
+    if script_args.main:
+        gen_structs.append('  {{ {0}_n, {0}_s, {0}_l }}, \\'.format("main"))
+
+    files = glob.glob(os.path.join(script_args.js_source_path, '*.js'))
+
+    for path in files:
+        if os.path.basename(path) not in script_args.ignore_files:
+            name = extract_name(path)
+            gen_output.append(js_to_native_code(path, name, script_args.build_type))
+            if name != 'main':
+                gen_structs.append('  {{ {0}_n, {0}_s, {0}_l }}, \\'.format(name))
+
+    gen_structs.append('  { NULL, NULL, 0 } \\\n};')
+
+    gen_output.append("\n".join(gen_structs))
+    gen_output.append(FOOTER)
+
+    with open(os.path.join(script_args.output_path, 'jerry-targetjs.h'), 'w') as gen_file:
+        gen_file.write("\n".join(gen_output))
+
+
+if __name__ == "__main__":
+    main()

tools/unicode_case_conversion.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 
 from __future__ import print_function
-from settings import PROJECT_DIR
-from unicode_c_source import Source
 
 import argparse
 import csv
@@ -25,6 +23,9 @@ import os
 import sys
 import warnings
 
+from settings import PROJECT_DIR
+from c_source_helper import UniCodeSource
+
 CONVERSIONS_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
 
 
@@ -69,7 +70,7 @@ def main():
     lower_case_conversions = conv_tables.get_lower_case_conversions()
     upper_case_conversions = conv_tables.get_upper_case_conversions()
 
-    c_source = Source(script_args.c_source)
+    c_source = UniCodeSource(script_args.c_source)
 
     unicode_file = os.path.basename(script_args.unicode_data)
     spec_casing_file = os.path.basename(script_args.special_casing)
@@ -395,8 +396,7 @@ def extract_character_pair_ranges(letter_case, reverse_letter_case):
             in_range = False
 
     # Remove all founded case mapping from the conversion tables after the scanning method
-    for idx in range(len(start_points)):
-        letter_id = start_points[idx]
+    for idx, letter_id in enumerate(start_points):
         conv_length = lengths[idx]
 
         for incr in range(0, conv_length, 2):

tools/unicode_ranges.py

@@ -27,8 +27,6 @@
 # separators:              Zs
 
 from __future__ import print_function
-from settings import PROJECT_DIR
-from unicode_c_source import Source
 
 import argparse
 import bisect
@@ -37,6 +35,9 @@ import itertools
 import os
 import sys
 
+from c_source_helper import UniCodeSource
+from settings import PROJECT_DIR
+
 RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
 
 def main():
@@ -65,7 +66,7 @@ def main():
     non_letter_tables = split_list(list(ranges(non_letters)))
     separator_tables = split_list(list(ranges(separators)))
 
-    c_source = Source(script_args.c_source)
+    c_source = UniCodeSource(script_args.c_source)
 
     header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
                          " * from %s. Do not edit! */" % os.path.basename(script_args.unicode_data),
@@ -181,9 +182,7 @@ def read_categories(unicode_data_file):
     separators = []
 
     with open(unicode_data_file) as unicode_data:
-        unicode_data_reader = csv.reader(unicode_data, delimiter=';')
-
-        for line in unicode_data_reader:
+        for line in csv.reader(unicode_data, delimiter=';'):
             unicode_id = int(line[0], 16)
 
             # Skip supplementary planes and ascii chars