YourWishes/jerryscript
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New unicode character handling using array data structure.

Browse Source 
JerryScript-DCO-1.0-Signed-off-by: István Kádár ikadar@inf.u-szeged.hu
This commit is contained in:
István Kádár
2016-02-15 16:31:48 +01:00
parent b8b587f76c
commit afa7b78b45
5 changed files with 464 additions and 4602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jerry-core/lit/lit-char-helpers.c
+119 -156
View File
@@ -14,9 +14,89 @@
*/
#include "lit-char-helpers.h"
#include "lit/lit-unicode-ranges.inc.h"
#include "lit-strings.h"
#define NUM_OF_ELEMENTS(array) (sizeof (array) / sizeof ((array)[0]))
/**
* Binary search algorithm that searches the a
* character in the given char array.
*
* @return true - if the character is in the given array
* false - otherwise
*/
static bool
search_char_in_char_array (ecma_char_t c, /**< code unit */
const ecma_char_t *array, /**< array */
int size_of_array) /**< length of the array */
{
int bottom = 0;
int top = size_of_array - 1;
while (bottom <= top)
{
int middle = (bottom + top) / 2;
ecma_char_t current = array[middle];
if (current == c)
{
return true;
}
if (c < current)
{
top = middle - 1;
}
else
{
bottom = middle + 1;
}
}
return false;
} /* search_char_in_char_array */
/**
* Binary search algorithm that searches a character in the given intervals.
* Intervals specifed by two arrays. The first one contains the starting points
* of the intervals, the second one contains the length of them.
*
* @return true - if the the character is included (inclusively) in one of the intervals in the given array
* false - otherwise
*/
static bool
search_char_in_interval_array (ecma_char_t c, /**< code unit */
const ecma_char_t *array_sp, /**< array of interval starting points */
const uint8_t *lengths, /**< array of interval lengths */
int size_of_array) /**< length of the array */
{
int bottom = 0;
int top = size_of_array - 1;
while (bottom <= top)
{
int middle = (bottom + top) / 2;
ecma_char_t current_sp = array_sp[middle];
if (current_sp <= c && c <= current_sp + lengths[middle])
{
return true;
}
if (c > current_sp)
{
bottom = middle + 1;
}
else
{
top = middle - 1;
}
}
return false;
} /* search_char_in_interval_array */
/**
* Check if specified character is one of the Format-Control characters
*
@@ -32,30 +112,8 @@ lit_char_is_format_control (ecma_char_t c) /**< code unit */
} /* lit_char_is_format_control */
/**
* Check if specified character is the Space Separator character
*
* See also:
* ECMA-262 v5, Table 2
*
* @return true - if the character falls into "Space, Separator" ("Zs") character category,
* false - otherwise.
*/
bool
lit_char_is_space_separator (ecma_char_t c) /**< code unit */
{
/* Zs */
#define LIT_UNICODE_RANGE_ZS(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
#include "lit-unicode-ranges.inc.h"
return false;
} /* lit_char_is_space_separator */
/**
* Check if specified character is one of the Whitespace characters
* Check if specified character is one of the Whitespace characters including those
* that fall into "Space, Separator" ("Zs") Unicode character category.
*
* @return true - if the character is one of characters, listed in ECMA-262 v5, Table 2,
* false - otherwise.
@@ -63,13 +121,21 @@ lit_char_is_space_separator (ecma_char_t c) /**< code unit */
bool
lit_char_is_white_space (ecma_char_t c) /**< code unit */
{
return (c == LIT_CHAR_TAB
|| c == LIT_CHAR_VTAB
|| c == LIT_CHAR_FF
|| c == LIT_CHAR_SP
|| c == LIT_CHAR_NBSP
|| c == LIT_CHAR_BOM
|| lit_char_is_space_separator (c));
if (c <= 127)
{
return (c == LIT_CHAR_TAB
|| c == LIT_CHAR_VTAB
|| c == LIT_CHAR_FF
|| c == LIT_CHAR_SP);
}
else
{
return (c == LIT_CHAR_NBSP
|| c == LIT_CHAR_BOM
|| (c >= unicode_separator_char_interv_sps[0]
&& c <= unicode_separator_char_interv_sps[0] + unicode_separator_char_interv_lens[0])
|| search_char_in_char_array (c, unicode_separator_chars, NUM_OF_ELEMENTS (unicode_separator_chars)));
}
} /* lit_char_is_white_space */
/**
@@ -119,60 +185,18 @@ lit_char_is_unicode_letter (ecma_char_t c) /**< code unit */
return false;
}
/* Lu */
#define LIT_UNICODE_RANGE_LU(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
/* Ll */
#define LIT_UNICODE_RANGE_LL(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
/* Lt */
#define LIT_UNICODE_RANGE_LT(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
/* Lm */
#define LIT_UNICODE_RANGE_LM(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
/* Lo */
#define LIT_UNICODE_RANGE_LO(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
/* Nl */
#define LIT_UNICODE_RANGE_NL(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
#include "lit-unicode-ranges.inc.h"
return false;
return (search_char_in_interval_array (c, unicode_letter_interv_sps, unicode_letter_interv_lens,
NUM_OF_ELEMENTS (unicode_letter_interv_sps))
|| search_char_in_char_array (c, unicode_letter_chars, NUM_OF_ELEMENTS (unicode_letter_chars)));
} /* lit_char_is_unicode_letter */
/**
* Check if specified character is a unicode combining mark
*
* Note:
* Unicode combining mark is a character, included into one of the following categories:
* - Non-spacing mark (Mn);
* - Combining spacing mark (Mc).
* Check if specified character is a non-letter character and can be used as a
* non-first character of an identifier.
* These characters coverd by the following unicode categories:
* - digit (Nd)
* - punctuation mark (Mn, Mc)
* - connector punctuation (Pc)
*
* See also:
* ECMA-262 v5, 7.6
@@ -181,82 +205,21 @@ lit_char_is_unicode_letter (ecma_char_t c) /**< code unit */
* false - otherwise.
*/
bool
lit_char_is_unicode_combining_mark (ecma_char_t c) /**< code unit */
lit_char_is_unicode_non_letter_ident_part (ecma_char_t c) /**< code unit */
{
/* Mn */
#define LIT_UNICODE_RANGE_MN(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
if (c <= 127)
{
return LIT_CHAR_ASCII_DIGITS_BEGIN <= c && c <= LIT_CHAR_ASCII_DIGITS_END;
}
/* Mc */
#define LIT_UNICODE_RANGE_MC(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
else
{
return (search_char_in_interval_array (c, unicode_non_letter_ident_part_interv_sps,
unicode_non_letter_ident_part_interv_lens,
NUM_OF_ELEMENTS (unicode_non_letter_ident_part_interv_sps))
|| search_char_in_char_array (c, unicode_non_letter_ident_part_chars,
NUM_OF_ELEMENTS (unicode_non_letter_ident_part_chars)));
}
#include "lit-unicode-ranges.inc.h"
return false;
} /* lit_char_is_unicode_combining_mark */
/**
* Check if specified character is a unicode digit
*
* Note:
* Unicode digit is a character, included into the following category:
* - Decimal number (Nd).
*
* See also:
* ECMA-262 v5, 7.6
*
* @return true - if specified character falls into the specified category,
* false - otherwise.
*/
bool
lit_char_is_unicode_digit (ecma_char_t c) /**< code unit */
{
/* Nd */
#define LIT_UNICODE_RANGE_ND(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
#include "lit-unicode-ranges.inc.h"
return false;
} /* lit_char_is_unicode_digit */
/**
* Check if specified character is a unicode connector punctuation
*
* Note:
* Unicode connector punctuation is a character, included into the following category:
* - Connector punctuation (Pc).
*
* See also:
* ECMA-262 v5, 7.6
*
* @return true - if specified character falls into the specified category,
* false - otherwise.
*/
bool
lit_char_is_unicode_connector_punctuation (ecma_char_t c) /**< code unit */
{
/* Pc */
#define LIT_UNICODE_RANGE_PC(range_begin, range_end) \
if (c >= (range_begin) && c <= (range_end)) \
{ \
return true; \
}
#include "lit-unicode-ranges.inc.h"
return false;
} /* lit_char_is_unicode_connector_punctuation */
} /* lit_char_is_unicode_non_letter_ident_part */
/**
* Check if specified character is one of OctalDigit characters (ECMA-262 v5, B.1.2)
jerry-core/lit/lit-char-helpers.h
+1 -4
View File
@@ -37,7 +37,6 @@ extern bool lit_char_is_format_control (ecma_char_t);
#define LIT_CHAR_NBSP ((ecma_char_t) 0x00A0) /* no-break space */
/* LIT_CHAR_BOM is defined above */
extern bool lit_char_is_space_separator (ecma_char_t);
extern bool lit_char_is_white_space (ecma_char_t);
/*
@@ -77,9 +76,7 @@ extern bool lit_char_is_line_terminator (ecma_char_t);
/* LIT_CHAR_BACKSLASH defined above */
extern bool lit_char_is_unicode_letter (ecma_char_t);
extern bool lit_char_is_unicode_combining_mark (ecma_char_t);
extern bool lit_char_is_unicode_digit (ecma_char_t);
extern bool lit_char_is_unicode_connector_punctuation (ecma_char_t);
extern bool lit_char_is_unicode_non_letter_ident_part (ecma_char_t);
/*
* Punctuator characters (ECMA-262 v5, 7.7)
jerry-core/lit/lit-unicode-ranges.inc.h
+179 -4321
View File
File diff suppressed because it is too large Load Diff
jerry-core/parser/js/common.c
+1 -3
View File
@@ -96,9 +96,7 @@ util_is_identifier_part_character (uint16_t chr) /**< EcmaScript character */
}
return (lit_char_is_unicode_letter (chr)
|| lit_char_is_unicode_combining_mark (chr)
|| lit_char_is_unicode_digit (chr)
|| lit_char_is_unicode_connector_punctuation (chr));
|| lit_char_is_unicode_non_letter_ident_part (chr));
} /* util_is_identifier_part_character */
/**
tools/print-unicode-ranges.sh
+164 -118
View File
@@ -17,124 +17,170 @@
#
# http://www.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.txt
#
# unicode categories: Lu Ll Lt Mn Mc Me Nd Nl No Zs Zl Zp Cc Cf Cs Co Lm Lo Pc Pd Ps Pe Pi Pf Po Sm Sc Sk So
# letter: Lu Ll Lt Lm Lo Nl
# non-letter-indent-part:
# digit: Nd
# punctuation mark: Mn Mc
# connector punctuation: Pc
# separators: Zs
if [ $# -le 4 ]; then
echo "useage: print-unicode-ranges.sh <unicode-data-path> <-i y sp|y len|n> <-cat letters|non-let-indent-parts|separators>"
echo " -i: y sp - print interval starting points"
echo " y len - print interval lengths"
echo " n - print individual characters"
echo " -cat: whether print letters|non-let-indent-parts|separators category"
exit 1
fi
STARTING_POINT="len"
UNICODE_DATA_PATH="$1"
shift
#
# One of unicode character category names (Lu, Ll, Nl, etc.)
#
UNICODE_CHAR_CATEGORY="$2"
while [ $# -gt 0 ]; do
if [ $1 == "-i" ]; then
shift
PRINT_INTERVALS="$1"
if [ $PRINT_INTERVALS == "y" ]; then
shift
STARTING_POINT="$1"
echo $STARTING_POINT
fi
elif [ $1 == "-cat" ]; then
shift
CATEGORY="$1"
echo $CATEGORY
fi
shift
done
UNICODE_CHAR_CATEGORY_UPPER_CASE=`echo $UNICODE_CHAR_CATEGORY | tr '[:lower:]' '[:upper:]'`
#
# 1. Print character codes, categories, and names
# 2. Filter by category
# 3. Print character codes and names without categories
# 4. Sort
# 5. Add '0x' to each line
# 6. Combine hexadecimal numbers into named ranges
# 7. Print ranges in format "LIT_UNICODE_RANGE_$UNICODE_CHAR_CATEGORY_UPPER_CASE (range_begin, range_end) /* range name */"
#
cut -d ';' "$UNICODE_DATA_PATH" -f 1,2,3 \
| grep ";$UNICODE_CHAR_CATEGORY\$" \
| cut -d ';' -f 1,2 \
| sort \
| awk 'BEGIN { FS=";"; OFS=";" } { print "0x"$1, $2; }' \
| awk --non-decimal-data \
'BEGIN \
{ \
FS=";"; \
OFS=";"; \
is_in_range=0; \
} \
\
function output_next_range () \
{ \
if (range_begin == range_prev) \
{ \
print range_begin, range_prev, range_begin_name; \
} \
else \
{ \
print range_begin, range_prev, range_begin_name, range_prev_name; \
} \
} \
\
{ \
if (is_in_range == 0) \
{ \
is_in_range=1; \
range_begin=$1; \
range_prev=$1; \
range_begin_name=$2; \
range_prev_name=$2; \
} \
else \
{ \
if (range_prev + 1 == $1) \
{ \
range_prev=$1; \
range_prev_name=$2
} \
else \
{ \
output_next_range(); \
range_begin=$1; \
range_prev=$1; \
range_begin_name=$2; \
range_prev_name=$2; \
} \
} \
} \
\
END \
{ \
output_next_range(); \
}' \
| awk \
'BEGIN \
{ \
FS = ";" \
} \
{ \
range_string = sprintf ("LIT_UNICODE_RANGE_'$UNICODE_CHAR_CATEGORY_UPPER_CASE' (%s, %s)", $1, $2); \
range_string_length = length (range_string); \
\
range_begin_name=$3; \
range_end_name=$4; \
\
range_begin_name_length = length (range_begin_name); \
range_end_name_length = length (range_end_name); \
\
printf "%s", range_string; \
if (range_end_name_length == 0) \
{ \
printf " /* %s */\n", range_begin_name; \
} \
else \
{ \
if (range_begin_name_length > range_end_name_length) \
{ \
indent1 = 0; \
indent2 = range_string_length + range_begin_name_length / 2;
indent3 = range_string_length + (range_begin_name_length - range_end_name_length) / 2; \
} \
else \
{ \
indent1 = (range_end_name_length - range_begin_name_length) / 2; \
indent2 = range_string_length + range_end_name_length / 2;
indent3 = range_string_length; \
} \
indent3 = indent3 + 3; \
fmt1 = sprintf (" /* %%%ds%%s\n", indent1); \
fmt2 = sprintf (" %%%ds<--->\n", indent2); \
fmt3 = sprintf (" %%%ds%%s */\n", indent3); \
\
printf fmt1, "", $3; \
printf fmt2, ""; \
printf fmt3, "", $4; \
} \
\
printf "\n"; \
}'
awk -v desired_category="$CATEGORY" \
'BEGIN \
{ \
FS=";"; OFS=";" \
} \
{ \
cat=$3; \
if (desired_category == "letters" && (cat == "Lu" || cat == "Ll" || cat == "Lt" || cat == "Lm" || cat == "Lo" || cat == "Nl")) \
{ \
print "0x"$1, $2, $3; \
} \
else if (desired_category == "non-let-indent-parts" && (cat == "Nd" || cat == "Mn" || cat == "Mc" || cat == "Pc")) \
{ \
print "0x"$1, $2, $3; \
} \
else if (desired_category == "separators" && cat == "Zs") \
{ \
print "0x"$1, $2, $3; \
} \
}' $UNICODE_DATA_PATH \
| gawk --non-decimal-data -v print_intervals="$PRINT_INTERVALS" -v sp="$STARTING_POINT" \
'BEGIN \
{ \
FS = ";"; \
OFS = ";"; \
is_in_range = 0; \
print_count = 0; \
} \
\
function print_Nl() \
{ \
++print_count; \
if (print_count == 10) \
{ \
printf "\n"; \
print_count = 0; \
} \
} \
\
function output_next_range () \
{ \
if (range_begin != range_prev && print_intervals=="y") \
{ \
i1 = strtonum(range_begin); \
i2 = strtonum(range_prev); \
len = i2 - i1; \
# if the length of an interval is > 255 have to spilt it into 255-lenth ones
if (len > 255) \
{ \
numOfSubintervals = (len / 255); # more precisely number of subintervals - 1 \
for (i = 1; i <= numOfSubintervals; ++i) \
{ \
if (sp == "sp") \
{ \
printf "0X%X, ", i1; \
print_Nl(); \
}
else \
{ \
printf "%d, ", 255; \
print_Nl(); \
} \
i1 = i1 + 256; # next interval begins on the ending of the previous + 1 \
} \
if (sp == "sp") \
{ \
printf "0X%X, ", i1; \
print_Nl(); \
} \
else \
{ \
printf "%d, ", len % 255 - (i-1); \
print_Nl(); \
} \
} \
else \
{ \
if (sp == "sp") \
{ \
printf "%s, ", range_begin; \
print_Nl(); \
} \
else \
{ \
printf "%d, ", len; \
print_Nl(); \
} \
} \
} \
else if (range_begin == range_prev && print_intervals != "y")\
{ \
printf "%s, ", range_begin; \
print_Nl(); \
} \
} \
\
{ \
if (is_in_range == 0) \
{ \
is_in_range = 1; \
range_begin = $1; \
range_prev = $1; \
range_begin_name = $2; \
range_prev_name = $2; \
} \
else \
{ \
if (range_prev + 1 == $1) \
{ \
range_prev = $1; \
range_prev_name = $2
} \
else \
{ \
output_next_range(); \
range_begin = $1; \
range_prev=$1; \
range_begin_name = $2; \
range_prev_name = $2; \
} \
} \
} \
\
END \
{ \
output_next_range(); \
}'