Refactor js2c.py and eliminate related pylint warnings (#1728)

JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com

targets/esp8266/Makefile.esp8266

@@ -92,7 +92,7 @@ jerry:
 
 
 js2c:
-	cd targets/esp8266; ../tools/js2c.py
+	cd targets/esp8266; ../../tools/js2c.py
 
 
 mkbin:

targets/mbed/Makefile.mbed

@@ -71,7 +71,7 @@ jerry:
 	cp $(BUILD_DIR)/lib/libjerry-libm.a $(COPYTARGET)/libjerrylibm.a
 
 js2c:
-	cd targets/mbed; ../tools/js2c.py;
+	cd targets/mbed; ../../tools/js2c.py;
 
 yotta:
 	cd targets/mbed; \

targets/mbedos5/Makefile

@@ -66,7 +66,7 @@ clean:
 	rm -rf .build/$(BOARD)
 
 js2c: js/main.js js/flash_leds.js
-	python ../tools/js2c.py --ignore pins.js
+	python ../../tools/js2c.py --ignore pins.js
 
 source/pins.cpp:
 	python tools/generate_pins.py ${BOARD}

targets/tools/js2c.py

@@ -1,154 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright JS Foundation and other contributors, http://js.foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#  This file converts ./js/*.js to a C-array in ./source/jerry-targetjs.h file
-
-import argparse
-import glob
-import os
-import re
-import sys
-
-special_chars = re.compile(r'[-\\?\'".]')
-
-def extractName(path):
-    return special_chars.sub('_', os.path.splitext(os.path.basename(path))[0])
-
-def writeLine(fo, content, indent=0):
-    buf = '  ' * indent + content + '\n'
-    fo.write(buf)
-
-def regroup(l, n):
-    return [ l[i:i+n] for i in range(0, len(l), n) ]
-
-def removeComments(code):
-    pattern = r'(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)'
-    regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
-    def _replacer(match):
-        if match.group(2) is not None:
-            return ""
-        else:
-            return match.group(1)
-    return regex.sub(_replacer, code)
-
-def removeWhitespaces(code):
-    return re.sub('\n+', '\n', re.sub('\n +', '\n', code))
-
-
-LICENSE = '''/* Copyright JS Foundation and other contributors, http://js.foundation
- *
- * Licensed under the Apache License, Version 2.0 (the \"License\");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an \"AS IS\" BASIS
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * This file is generated by js2c.py. Please do not modify.
- */
-
-'''
-
-HEADER = '''#ifndef JERRY_TARGETJS_H
-#define JERRY_TARGETJS_H
-
-'''
-
-FOOTER = '''
-#endif
-
-'''
-
-parser = argparse.ArgumentParser(description="js2c")
-parser.add_argument('--build-type', help='build type', default='release', choices=['release', 'debug'])
-parser.add_argument('--ignore', help='files to ignore', dest='ignore_files', default=[], action='append')
-parser.add_argument('--no-main', help="don't require a 'main.js' file", dest='main', action='store_false', default=True)
-parser.add_argument('--js-source', dest='js_source_path', default='./js', help='Source directory of JavaScript files" (default: %(default)s)')
-parser.add_argument('--dest', dest='output_path', default='./source', help="Destination directory of 'jerry-targetjs.h' (default: %(default)s)")
-
-args = parser.parse_args()
-
-# argument processing
-build_type = args.build_type
-ignore_files = args.ignore_files
-
-fout = open(os.path.join(args.output_path, 'jerry-targetjs.h'), 'w')
-fout.write(LICENSE);
-fout.write(HEADER);
-
-def exportOneFile(path, name):
-    fout.write('const static char ' + name + '_n[] = "' + name + '";\n')
-    fout.write('const static char ' + name + '_s[] =\n{\n')
-
-    fin = open(path, 'r');
-    code = fin.read() + '\0'
-
-    # minimize code when release mode
-    if build_type != 'debug':
-        code = removeComments(code)
-        code = removeWhitespaces(code)
-
-    for line in regroup(code, 10):
-        buf = ', '.join(map(lambda ch: format(ord(ch),"#04x"), line))
-        if line[-1] != '\0':
-            buf += ','
-        writeLine(fout, buf, 1)
-    writeLine(fout, '};')
-    writeLine(fout, 'const static int ' + name + '_l = ' + str(len(code)-1) + ';')
-    writeLine(fout, '')
-
-    fin.close();
-
-def exportOneName(name):
-    writeLine(fout, '{ ' + name + '_n, ' + name + '_s, ' + name + '_l }, \\', 1)
-
-files = glob.glob(os.path.join(args.js_source_path, '*.js'))
-for path in files:
-    name = extractName(path)
-    if os.path.basename(path) not in ignore_files:
-        exportOneFile(path, name)
-
-
-NATIVE_STRUCT = '''
-struct js_source_all {
-  const char* name;
-  const char* source;
-  const int length;
-};
-
-#define DECLARE_JS_CODES \\
-struct js_source_all js_codes[] = \\
-{ \\
-'''
-
-fout.write(NATIVE_STRUCT)
-if args.main:
-    exportOneName('main')
-for filename in files:
-    name = extractName(filename)
-    if name != 'main' and os.path.basename(filename) not in ignore_files:
-        exportOneName(name)
-
-writeLine(fout, '{ NULL, NULL, 0 } \\', 1)
-writeLine(fout, '};')
-
-fout.write(FOOTER)
-fout.close()

tools/c_source_helper.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 LICENSE = """/* Copyright JS Foundation and other contributors, http://js.foundation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,7 +31,7 @@ LICENSE = """/* Copyright JS Foundation and other contributors, http://js.founda
  */"""
 
 
-class Source(object):
+class UniCodeSource(object):
     def __init__(self, filepath):
         self.__filepath = filepath
         self.__header = [LICENSE, ""]
@@ -38,7 +39,7 @@ class Source(object):
 
     def complete_header(self, completion):
         self.__header.append(completion)
-        self.__header.append("") # for an extra empty line
+        self.__header.append("")  # for an extra empty line
 
     def add_table(self, table, table_name, table_type, table_descr):
         self.__data.append(table_descr)
@@ -46,30 +47,33 @@ class Source(object):
         self.__data.append("{")
         self.__data.append(format_code(table, 1))
         self.__data.append("};")
-        self.__data.append("") # for an extra empty line
+        self.__data.append("")  # for an extra empty line
 
     def generate(self):
-        with open(self.__filepath, 'w') as genereted_source:
+        with open(self.__filepath, 'w') as generated_source:
-            genereted_source.write("\n".join(self.__header))
+            generated_source.write("\n".join(self.__header))
-            genereted_source.write("\n".join(self.__data))
+            generated_source.write("\n".join(self.__data))
 
 
 def regroup(list_to_group, num):
     return [list_to_group[i:i+num] for i in range(0, len(list_to_group), num)]
 
 
-def hex_format(char):
+def hex_format(char, digit_number):
     if isinstance(char, str):
         char = ord(char)
 
-    return "0x{:04x}".format(char)
+    return ("0x{:0%sx}" % digit_number).format(char)
 
 
-def format_code(code, indent):
+def format_code(code, indent, digit_number=4):
     lines = []
+
+    nums_per_line = 10
+    width = nums_per_line * (digit_number + 4)
     # convert all characters to hex format
-    converted_code = [hex_format(char) for char in code]
+    converted_code = [hex_format(char, digit_number) for char in code]
     # 10 hex number per line
-    for line in regroup(", ".join(converted_code), 10 * 8):
+    for line in regroup(", ".join(converted_code), width):
         lines.append(('  ' * indent) + line.strip())
     return "\n".join(lines)

tools/js2c.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#  This file converts ./js/*.js to a C-array in ./source/jerry-targetjs.h file
+
+import argparse
+import glob
+import os
+import re
+import c_source_helper
+
+HEADER = '''#ifndef JERRY_TARGETJS_H
+#define JERRY_TARGETJS_H
+'''
+
+FOOTER = '''
+#endif
+'''
+
+NATIVE_STRUCT = '''
+struct js_source_all {
+  const char* name;
+  const char* source;
+  const int length;
+};
+
+#define DECLARE_JS_CODES \\
+struct js_source_all js_codes[] = \\
+{ \\'''
+
+
+def extract_name(path):
+    special_chars = re.compile(r'[-\\?\'".]')
+    return special_chars.sub('_', os.path.splitext(os.path.basename(path))[0])
+
+
+def reduce_code(code):
+    code = re.sub(r"/\*.*?\*/", "", code, flags=re.DOTALL)  # remove all occurance streamed comments
+    code = re.sub(r"//.*?\n", "", code)  # remove all occurance singleline comments
+    code = re.sub('\n+', '\n', re.sub('\n +', '\n', code))  # remove white spaces
+    return code
+
+
+def js_to_native_code(path, name, build_type):
+    with open(path, 'r') as js_source:
+        code = js_source.read()
+
+    if build_type != 'debug':
+        code = reduce_code(code)
+
+    data = c_source_helper.format_code(code, 1, 2)
+
+    native_code = """const static char {0}_n[] = "{0}";
+const static char {0}_s[] =
+{{
+{1}
+}};
+const static int {0}_l = {2};
+""".format(name, data, len(code))
+
+    return native_code
+
+
+def main():
+    parser = argparse.ArgumentParser(description="js2c")
+    parser.add_argument('--build-type', help='build type', default='release', choices=['release', 'debug'])
+    parser.add_argument('--ignore', help='files to ignore', dest='ignore_files', default=[], action='append')
+    parser.add_argument('--no-main',
+                        help="don't require a 'main.js' file",
+                        dest='main',
+                        action='store_false',
+                        default=True)
+    parser.add_argument('--js-source',
+                        dest='js_source_path',
+                        default='./js',
+                        help='Source directory of JavaScript files" (default: %(default)s)')
+    parser.add_argument('--dest',
+                        dest='output_path',
+                        default='./source',
+                        help="Destination directory of 'jerry-targetjs.h' (default: %(default)s)")
+
+    script_args = parser.parse_args()
+
+    gen_line = "/* This file is generated by %s. Please do not modify. */" % os.path.basename(__file__)
+
+    gen_output = [c_source_helper.LICENSE, "", gen_line, "", HEADER]
+    gen_structs = [NATIVE_STRUCT]
+
+    if script_args.main:
+        gen_structs.append('  {{ {0}_n, {0}_s, {0}_l }}, \\'.format("main"))
+
+    files = glob.glob(os.path.join(script_args.js_source_path, '*.js'))
+
+    for path in files:
+        if os.path.basename(path) not in script_args.ignore_files:
+            name = extract_name(path)
+            gen_output.append(js_to_native_code(path, name, script_args.build_type))
+            if name != 'main':
+                gen_structs.append('  {{ {0}_n, {0}_s, {0}_l }}, \\'.format(name))
+
+    gen_structs.append('  { NULL, NULL, 0 } \\\n};')
+
+    gen_output.append("\n".join(gen_structs))
+    gen_output.append(FOOTER)
+
+    with open(os.path.join(script_args.output_path, 'jerry-targetjs.h'), 'w') as gen_file:
+        gen_file.write("\n".join(gen_output))
+
+
+if __name__ == "__main__":
+    main()

tools/unicode_case_conversion.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 
 from __future__ import print_function
-from settings import PROJECT_DIR
-from unicode_c_source import Source
 
 import argparse
 import csv
@@ -25,6 +23,9 @@ import os
 import sys
 import warnings
 
+from settings import PROJECT_DIR
+from c_source_helper import UniCodeSource
+
 CONVERSIONS_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
 
 
@@ -69,7 +70,7 @@ def main():
     lower_case_conversions = conv_tables.get_lower_case_conversions()
     upper_case_conversions = conv_tables.get_upper_case_conversions()
 
-    c_source = Source(script_args.c_source)
+    c_source = UniCodeSource(script_args.c_source)
 
     unicode_file = os.path.basename(script_args.unicode_data)
     spec_casing_file = os.path.basename(script_args.special_casing)
@@ -395,8 +396,7 @@ def extract_character_pair_ranges(letter_case, reverse_letter_case):
             in_range = False
 
     # Remove all founded case mapping from the conversion tables after the scanning method
-    for idx in range(len(start_points)):
+    for idx, letter_id in enumerate(start_points):
-        letter_id = start_points[idx]
         conv_length = lengths[idx]
 
         for incr in range(0, conv_length, 2):

tools/unicode_ranges.py

@@ -27,8 +27,6 @@
 # separators:              Zs
 
 from __future__ import print_function
-from settings import PROJECT_DIR
-from unicode_c_source import Source
 
 import argparse
 import bisect
@@ -37,6 +35,9 @@ import itertools
 import os
 import sys
 
+from c_source_helper import UniCodeSource
+from settings import PROJECT_DIR
+
 RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
 
 def main():
@@ -65,7 +66,7 @@ def main():
     non_letter_tables = split_list(list(ranges(non_letters)))
     separator_tables = split_list(list(ranges(separators)))
 
-    c_source = Source(script_args.c_source)
+    c_source = UniCodeSource(script_args.c_source)
 
     header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
                          " * from %s. Do not edit! */" % os.path.basename(script_args.unicode_data),
@@ -181,9 +182,7 @@ def read_categories(unicode_data_file):
     separators = []
 
     with open(unicode_data_file) as unicode_data:
-        unicode_data_reader = csv.reader(unicode_data, delimiter=';')
+        for line in csv.reader(unicode_data, delimiter=';'):
-
-        for line in unicode_data_reader:
             unicode_id = int(line[0], 16)
 
             # Skip supplementary planes and ascii chars