Refactor the generator scripts for unicode tables (#1623)

Extract the source code generator methods into a separated `unicode_c_source.py` script.
Fix the generator scripts to make them compatible with both Python2 and Python3.
Remove pylint warnings.

JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com

jerry-core/lit/lit-unicode-conversions.inc.h

@@ -11,10 +11,11 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
- * This file is automatically generated by the unicode_case_conversion.py script. Do not edit!
  */
 
+/* This file is automatically generated by the unicode_case_conversion.py script
+ * from UnicodeData-9.0.0.txt and SpecialCasing-9.0.0.txt files. Do not edit! */
+
 /* Contains start points of character case ranges (these are bidirectional conversions). */
 static const uint16_t jerry_character_case_ranges[] JERRY_CONST_DATA =
 {
@@ -154,9 +155,8 @@ static const uint16_t jerry_upper_case_conversions[] JERRY_CONST_DATA =
   0x0046, 0x004c
 };
 
-/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
+/* Number of one-to-one, one-to-two, and one-to-three uppercase conversions. */
 static const uint8_t jerry_upper_case_conversion_counters[] JERRY_CONST_DATA =
 {
   0x001c, 0x002c, 0x0010
 };
-

jerry-core/lit/lit-unicode-ranges.inc.h

@@ -11,11 +11,11 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
- * This file is automatically generated by the unicode_ranges.py script
- * from UnicodeData-3.0.0.txt. Do not edit!
  */
 
+/* This file is automatically generated by the unicode_ranges.py script
+ * from UnicodeData-3.0.0.txt. Do not edit! */
+
 /**
  * Character interval starting points for the unicode letters.
  *
@@ -180,4 +180,3 @@ static const uint16_t jerry_unicode_separator_chars[] JERRY_CONST_DATA =
 {
   0x1680, 0x180e, 0x202f, 0x205f, 0x3000
 };
-

tools/unicode_c_source.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LICENSE = """/* Copyright JS Foundation and other contributors, http://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */"""
+
+
+class Source(object):
+    def __init__(self, filepath):
+        self.__filepath = filepath
+        self.__header = [LICENSE, ""]
+        self.__data = []
+
+    def complete_header(self, completion):
+        self.__header.append(completion)
+        self.__header.append("") # for an extra empty line
+
+    def add_table(self, table, table_name, table_type, table_descr):
+        self.__data.append(table_descr)
+        self.__data.append("static const %s jerry_%s[] JERRY_CONST_DATA =" % (table_type, table_name))
+        self.__data.append("{")
+        self.__data.append(format_code(table, 1))
+        self.__data.append("};")
+        self.__data.append("") # for an extra empty line
+
+    def generate(self):
+        with open(self.__filepath, 'w') as genereted_source:
+            genereted_source.write("\n".join(self.__header))
+            genereted_source.write("\n".join(self.__data))
+
+
+def regroup(list_to_group, num):
+    return [list_to_group[i:i+num] for i in range(0, len(list_to_group), num)]
+
+
+def hex_format(char):
+    if isinstance(char, str):
+        char = ord(char)
+
+    return "0x{:04x}".format(char)
+
+
+def format_code(code, indent):
+    lines = []
+    # convert all characters to hex format
+    converted_code = [hex_format(char) for char in code]
+    # 10 hex number per line
+    for line in regroup(", ".join(converted_code), 10 * 8):
+        lines.append(('  ' * indent) + line.strip())
+    return "\n".join(lines)

tools/unicode_ranges.py

@@ -26,152 +26,146 @@
 #   connector punctuation: Pc
 # separators:              Zs
 
+from __future__ import print_function
+from settings import PROJECT_DIR
+from unicode_c_source import Source
+
 import argparse
 import bisect
 import csv
 import itertools
 import os
+import sys
 
-TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
-PROJECT_DIR = os.path.normpath(os.path.join(TOOLS_DIR, '..'))
-C_SOURCE_FILE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument('unicode_data',
-                    metavar='FILE',
-                    action='store',
-                    help='specify the unicode data file')
-
-parser.add_argument('--c-source',
-                    metavar='FILE',
-                    action='store',
-                    default=C_SOURCE_FILE,
-                    help='specify the output c source (default: %(default)s)')
-
-script_args = parser.parse_args()
-
 
 def main():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('unicode_data',
+                        metavar='FILE',
+                        action='store',
+                        help='specify the unicode data file')
+
+    parser.add_argument('--c-source',
+                        metavar='FILE',
+                        action='store',
+                        default=RANGES_C_SOURCE,
+                        help='specify the output c source (default: %(default)s)')
+
+    script_args = parser.parse_args()
+
     if not os.path.isfile(script_args.unicode_data) or not os.access(script_args.unicode_data, os.R_OK):
         print('The %s file is missing or not readable!' % script_args.unicode_data)
         sys.exit(1)
 
-    letters, non_letters, separators = read_categories()
+    letters, non_letters, separators = read_categories(script_args.unicode_data)
 
-    letters_list = list(ranges(letters))
+    letter_tables = split_list(list(ranges(letters)))
-    letter_interval_sps, letter_interval_lengths, letter_chars = split_list(letters_list)
+    non_letter_tables = split_list(list(ranges(non_letters)))
+    separator_tables = split_list(list(ranges(separators)))
 
-    non_letters_list = list(ranges(non_letters))
+    c_source = Source(script_args.c_source)
-    non_letter_interval_sps, non_letter_interval_lengths, non_letter_chars = split_list(non_letters_list)
 
-    separator_list = list(ranges(separators))
+    header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
-    separator_interval_sps, separator_interval_lengths, separator_chars = split_list(separator_list)
+                         " * from %s. Do not edit! */" % os.path.basename(script_args.unicode_data),
+                         ""]
 
-    source = GenSource()
+    c_source.complete_header("\n".join(header_completion))
 
-    letter_interval_sps_desc = """/**
+    c_source.add_table(letter_tables[0],
- * Character interval starting points for the unicode letters.
+                       "unicode_letter_interval_sps",
- *
+                       "uint16_t",
- * The characters covered by these intervals are from
+                       ("/**\n"
- * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl
+                        " * Character interval starting points for the unicode letters.\n"
- */"""
+                        " *\n"
-    source.add_table("uint16_t",
+                        " * The characters covered by these intervals are from\n"
-                     "unicode_letter_interval_sps",
+                        " * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl\n"
-                     letter_interval_sps,
+                        " */"))
-                     letter_interval_sps_desc)
 
-    letter_interval_lengths_desc = """/**
+    c_source.add_table(letter_tables[1],
- * Character lengths for the unicode letters.
+                       "unicode_letter_interval_lengths",
- *
+                       "uint8_t",
- * The characters covered by these intervals are from
+                       ("/**\n"
- * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl
+                        " * Character lengths for the unicode letters.\n"
- */"""
+                        " *\n"
-    source.add_table("uint8_t",
+                        " * The characters covered by these intervals are from\n"
-                     "unicode_letter_interval_lengths",
+                        " * the following Unicode categories: Lu, Ll, Lt, Lm, Lo, Nl\n"
-                     letter_interval_lengths,
+                        " */"))
-                     letter_interval_lengths_desc)
 
-    letter_chars_desc = """/**
+    c_source.add_table(letter_tables[2],
- * Those unicode letter characters that are not inside any of
+                       "unicode_letter_chars",
- * the intervals specified in jerry_unicode_letter_interval_sps array.
+                       "uint16_t",
- *
+                       ("/**\n"
- * The characters are from the following Unicode categories:
+                        " * Those unicode letter characters that are not inside any of\n"
- * Lu, Ll, Lt, Lm, Lo, Nl
+                        " * the intervals specified in jerry_unicode_letter_interval_sps array.\n"
- */"""
+                        " *\n"
-    source.add_table("uint16_t",
+                        " * The characters are from the following Unicode categories:\n"
-                     "unicode_letter_chars",
+                        " * Lu, Ll, Lt, Lm, Lo, Nl\n"
-                     letter_chars,
+                        " */"))
-                     letter_chars_desc)
 
-    non_letter_interval_sps_desc = """/**
+    c_source.add_table(non_letter_tables[0],
- * Character interval starting points for non-letter character
+                       "unicode_non_letter_ident_part_interval_sps",
- * that can be used as a non-first character of an identifier.
+                       "uint16_t",
- *
+                       ("/**\n"
- * The characters covered by these intervals are from
+                        " * Character interval starting points for non-letter character\n"
- * the following Unicode categories: Nd, Mn, Mc, Pc
+                        " * that can be used as a non-first character of an identifier.\n"
- */"""
+                        " *\n"
-    source.add_table("uint16_t",
+                        " * The characters covered by these intervals are from\n"
-                     "unicode_non_letter_ident_part_interval_sps",
+                        " * the following Unicode categories: Nd, Mn, Mc, Pc\n"
-                     non_letter_interval_sps,
+                        " */"))
-                     non_letter_interval_sps_desc)
 
-    non_letter_interval_lengths_desc = """/**
+    c_source.add_table(non_letter_tables[1],
- * Character interval lengths for non-letter character
+                       "unicode_non_letter_ident_part_interval_lengths",
- * that can be used as a non-first character of an identifier.
+                       "uint8_t",
- *
+                       ("/**\n"
- * The characters covered by these intervals are from
+                        " * Character interval lengths for non-letter character\n"
- * the following Unicode categories: Nd, Mn, Mc, Pc
+                        " * that can be used as a non-first character of an identifier.\n"
- */"""
+                        " *\n"
-    source.add_table("uint8_t",
+                        " * The characters covered by these intervals are from\n"
-                     "unicode_non_letter_ident_part_interval_lengths",
+                        " * the following Unicode categories: Nd, Mn, Mc, Pc\n"
-                     non_letter_interval_lengths,
+                        " */"))
-                     non_letter_interval_lengths_desc)
 
-    non_letter_chars_desc = """/**
+    c_source.add_table(non_letter_tables[2],
- * Those non-letter characters that can be used as a non-first
+                       "unicode_non_letter_ident_part_chars",
- * character of an identifier and not included in any of the intervals
+                       "uint16_t",
- * specified in jerry_unicode_non_letter_ident_part_interval_sps array.
+                       ("/**\n"
- *
+                        " * Those non-letter characters that can be used as a non-first\n"
- * The characters are from the following Unicode categories:
+                        " * character of an identifier and not included in any of the intervals\n"
- * Nd, Mn, Mc, Pc
+                        " * specified in jerry_unicode_non_letter_ident_part_interval_sps array.\n"
- */"""
+                        " *\n"
-    source.add_table("uint16_t",
+                        " * The characters are from the following Unicode categories:\n"
-                     "unicode_non_letter_ident_part_chars",
+                        " * Nd, Mn, Mc, Pc\n"
-                     non_letter_chars,
+                        " */"))
-                     non_letter_chars_desc)
 
-    separator_interval_sps_desc = """/**
+    c_source.add_table(separator_tables[0],
- * Unicode separator character interval starting points from Unicode category: Zs
+                       "unicode_separator_char_interval_sps",
- */"""
+                       "uint16_t",
-    source.add_table("uint16_t",
+                       ("/**\n"
-                     "unicode_separator_char_interval_sps",
+                        " * Unicode separator character interval starting points from Unicode category: Zs\n"
-                     separator_interval_sps,
+                        " */"))
-                     separator_interval_sps_desc)
 
-    separator_interval_lengths_desc = """/**
+    c_source.add_table(separator_tables[1],
- * Unicode separator character interval lengths from Unicode category: Zs
+                       "unicode_separator_char_interval_lengths",
- */"""
+                       "uint8_t",
-    source.add_table("uint8_t",
+                       ("/**\n"
-                     "unicode_separator_char_interval_lengths",
+                        " * Unicode separator character interval lengths from Unicode category: Zs\n"
-                     separator_interval_lengths,
+                        " */"))
-                     separator_interval_lengths_desc)
 
-    separator_chars_desc = """/**
+    c_source.add_table(separator_tables[2],
- * Unicode separator characters that are not in the
+                       "unicode_separator_chars",
- * jerry_unicode_separator_char_intervals array.
+                       "uint16_t",
- *
+                       ("/**\n"
- * Unicode category: Zs
+                        " * Unicode separator characters that are not in the\n"
- */"""
+                        " * jerry_unicode_separator_char_intervals array.\n"
-    source.add_table("uint16_t",
+                        " *\n"
-                     "unicode_separator_chars",
+                        " * Unicode category: Zs\n"
-                     separator_chars,
+                        " */"))
-                     separator_chars_desc)
 
-    source.write_source()
+    c_source.generate()
 
 
-def read_categories():
+def read_categories(unicode_data_file):
     """
     Read the corresponding unicode values and store them in category lists.
 
@@ -186,7 +180,7 @@ def read_categories():
     non_letters = []
     separators = []
 
-    with open(script_args.unicode_data) as unicode_data:
+    with open(unicode_data_file) as unicode_data:
         unicode_data_reader = csv.reader(unicode_data, delimiter=';')
 
         for line in unicode_data_reader:
@@ -228,10 +222,9 @@ def ranges(i):
 
     :return: List of ranges.
     """
-
+    for _, group in itertools.groupby(enumerate(i), lambda q: (q[1] - q[0])):
-    for a, b in itertools.groupby(enumerate(i), lambda (x, y): y - x):
+        group = list(group)
-        b = list(b)
+        yield group[0][1], group[-1][1]
-        yield b[0][1], b[-1][1]
 
 
 def split_list(category_list):
@@ -241,87 +234,24 @@ def split_list(category_list):
     :return: List of interval starting points, interval lengths and single chars
     """
 
-    unicode_category_interval_sps = []
+    interval_sps = []
-    unicode_category_interval_lengths = []
+    interval_lengths = []
-    unicode_category_chars = []
+    chars = []
 
     for element in category_list:
         interval_length = element[1] - element[0]
         if interval_length == 0:
-            unicode_category_chars.append(element[0])
+            chars.append(element[0])
-
+        elif interval_length > 255:
-        elif (interval_length > 255):
             for i in range(element[0], element[1], 256):
                 length = 255 if (element[1] - i > 255) else (element[1] - i)
-                unicode_category_interval_sps.append(i)
+                interval_sps.append(i)
-                unicode_category_interval_lengths.append(length)
+                interval_lengths.append(length)
         else:
-            unicode_category_interval_sps.append(element[0])
+            interval_sps.append(element[0])
-            unicode_category_interval_lengths.append(element[1] - element[0])
+            interval_lengths.append(element[1] - element[0])
 
-    return unicode_category_interval_sps, unicode_category_interval_lengths, unicode_category_chars
+    return interval_sps, interval_lengths, chars
-
-
-class GenSource(object):
-    """Class defines a default generated c source."""
-
-    def __init__(self):
-        self._data = []
-
-        header = """/* Copyright JS Foundation and other contributors, http://js.foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * This file is automatically generated by the {SCRIPT} script
- * from {UNICODES}. Do not edit!
- */
-
-""".format(SCRIPT=os.path.basename(__file__), UNICODES=os.path.basename(script_args.unicode_data))
-
-        self._data.append(header)
-
-    def _regroup(self, l, n):
-        return [l[i:i+n] for i in range(0, len(l), n)]
-
-    def _hex_format(self, ch):
-        if isinstance(ch, str):
-            ch = ord(ch)
-
-        return "0x{:04x}".format(ch)
-
-    def _format_code(self, code, indent):
-        lines = []
-        # convert all characters to hex format
-        converted_code = map(self._hex_format, code)
-        # 10 hex number per line
-        for line in self._regroup(", ".join(converted_code), 10 * 8):
-            lines.append(('  ' * indent) + line.strip())
-        return "\n".join(lines)
-
-    def add_table(self, type_name, array_name, table, description=""):
-        table_str = """{DESC}
-static const {TYPE} jerry_{NAME}[] JERRY_CONST_DATA =
-{{
-{TABLE}
-}};
-
-""".format(DESC=description, TYPE=type_name, NAME=array_name, TABLE=self._format_code(table, 1))
-
-        self._data.append(table_str)
-
-    def write_source(self):
-        with open(script_args.c_source, 'w') as genereted_source:
-            genereted_source.write(''.join(self._data))
 
 
 if __name__ == "__main__":