dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / web_unicode / casing.lucb
#==============================================================================================
#
# casing - full case mapping, case folding, case closure and width mapping
#
# DESCRIPTION:
# Not a port: replaces ICU's ucase.cpp and ustrcase.cpp (u_strToLower, u_strToUpper,
# u_strToTitle, u_strFoldCase, UnicodeSet::closeOver(USET_CASE_INSENSITIVE)) and the
# Halfwidth-Fullwidth transliterator, over the tables of generated_ucd_casing.lucb.
# The context conditions of SpecialCasing.txt (Final_Sigma, More_Above, ...) and the
# case locales of ICU (Turkish/Azeri, Lithuanian, Dutch titlecasing, Armenian) are ICU's.
# ICU's special uppercasing of Greek (removing accents) is not implemented: Greek text
# is uppercased as in the root locale.
#
#==============================================================================================
# mark: Case locales ============================================================================
## ICU's UCASE_LOC_*: the languages whose case mappings differ from the root locale's.
enum CaseLocale:
root
turkish
lithuanian
greek
dutch
armenian
## ucase_getCaseLocale: the case locale of a locale identifier's language subtag.
func case_locale_of(locale: const u8[]) -> CaseLocale:
var end: usize = 0
while end < locale.length and locale[end] != '-' and locale[end] != '_':
end += 1
let language = locale[..<end]
if ascii_equals_ignoring_case(language, "tr") or ascii_equals_ignoring_case(language, "tur") or ascii_equals_ignoring_case(language, "az") or ascii_equals_ignoring_case(language, "aze"):
return CaseLocale.turkish
if ascii_equals_ignoring_case(language, "lt") or ascii_equals_ignoring_case(language, "lit"):
return CaseLocale.lithuanian
if ascii_equals_ignoring_case(language, "el") or ascii_equals_ignoring_case(language, "ell"):
return CaseLocale.greek
if ascii_equals_ignoring_case(language, "nl") or ascii_equals_ignoring_case(language, "nld"):
return CaseLocale.dutch
if ascii_equals_ignoring_case(language, "hy") or ascii_equals_ignoring_case(language, "hye"):
return CaseLocale.armenian
return CaseLocale.root
func ascii_equals_ignoring_case(bytes: const u8[], text: str) -> bool:
if bytes.length != text.length:
return false
for index in 0..<bytes.length:
var a = bytes[index]
if a >= 'A' and a <= 'Z':
a += 32
if a != text.bytes[index]:
return false
return true
# mark: Case mapping tables =====================================================================
let case_mapping_digits: usize = 50
## The digit offset of the case mapping entry of `code_point`, or none.
func case_mapping_entry(code_point: u32) -> usize?:
var low: usize = 0
var high = ucd_case_mappings.length // case_mapping_digits
while low < high:
let middle = low + (high - low) // 2
let key = ucd_hex(ucd_case_mappings, middle * case_mapping_digits, 6)
if key < code_point:
low = middle + 1
elif key > code_point:
high = middle
else:
return middle * case_mapping_digits
return none
# The fields of a case mapping entry.
let case_simple_lower: usize = 6
let case_simple_upper: usize = 12
let case_simple_title: usize = 18
let case_simple_fold: usize = 24
let case_full_lower: usize = 30
let case_full_upper: usize = 35
let case_full_title: usize = 40
let case_full_fold: usize = 45
## A simple (single code point) case mapping: Simple_Lowercase_Mapping and friends.
func simple_case_mapping(code_point: u32, field: usize) -> u32:
let entry = case_mapping_entry(code_point) else return code_point
return ucd_hex(ucd_case_mappings, entry + field, 6)
## u_foldCase(c, U_FOLD_CASE_DEFAULT).
func simple_case_folding(code_point: u32) -> u32:
return simple_case_mapping(code_point, case_simple_fold)
## Append `length` code points of ucd_case_pool at `offset`.
func append_case_pool(offset: u32, length: u32, out: CodePointBuffer*):
for index in 0..<(usize)length:
out.append(ucd_hex(ucd_case_pool, ((usize)offset + index) * 6, 6))
## Append the unconditional full mapping of `full` (else the simple mapping of `simple`).
func append_full_case_mapping(code_point: u32, full: usize, simple: usize, out: CodePointBuffer*):
let found = case_mapping_entry(code_point)
if found == none:
out.append(code_point)
return
let entry = found else return
let length = ucd_hex(ucd_case_mappings, entry + full + 4, 1)
if length == 0:
out.append(ucd_hex(ucd_case_mappings, entry + simple, 6))
return
append_case_pool(ucd_hex(ucd_case_mappings, entry + full, 4), length, out)
# mark: Contexts ================================================================================
# ICU's dot types (ucase.h UCASE_DOT_MASK).
let dot_no_dot: u32 = 0
let dot_soft_dotted: u32 = 1
let dot_above: u32 = 2
let dot_other_accent: u32 = 3
func dot_type(code_point: u32) -> u32:
if ucd_has_binary_property(code_point, uchar_soft_dotted):
return dot_soft_dotted
let ccc = ucd_combining_class(code_point)
if ccc == 230:
return dot_above
if ccc != 0:
return dot_other_accent
return dot_no_dot
## isFollowedByCasedLetter: skipping case-ignorable characters from index + direction,
## whether the first other character is cased.
func is_followed_by_cased_letter(text: const u32[], index: usize, direction: isize) -> bool:
var position = (isize)index + direction
while position >= 0 and position < (isize)text.length:
let code_point = text[(usize)position]
if not ucd_has_binary_property(code_point, uchar_case_ignorable):
return ucd_has_binary_property(code_point, uchar_cased)
position += direction
return false
## isPrecededBySoftDotted.
func is_preceded_by_soft_dotted(text: const u32[], index: usize) -> bool:
var position = index
while position > 0:
position -= 1
let type_ = dot_type(text[position])
if type_ == dot_soft_dotted:
return true
if type_ != dot_other_accent:
return false
return false
## isPrecededBy_I.
func is_preceded_by_i(text: const u32[], index: usize) -> bool:
var position = index
while position > 0:
position -= 1
if text[position] == 0x49:
return true
if dot_type(text[position]) != dot_other_accent:
return false
return false
## isFollowedByMoreAbove.
func is_followed_by_more_above(text: const u32[], index: usize) -> bool:
for position in index + 1..<text.length:
let type_ = dot_type(text[position])
if type_ == dot_above:
return true
if type_ != dot_other_accent:
return false
return false
## isFollowedByDotAbove.
func is_followed_by_dot_above(text: const u32[], index: usize) -> bool:
for position in index + 1..<text.length:
if text[position] == 0x307:
return true
if dot_type(text[position]) != dot_other_accent:
return false
return false
# The conditions of ucd_conditional_case_mappings.
let condition_none: u32 = 0
let condition_final_sigma: u32 = 1
let condition_after_soft_dotted: u32 = 2
let condition_more_above: u32 = 3
let condition_before_dot: u32 = 4
let condition_after_i: u32 = 5
let condition_not: u32 = 8
func case_condition_holds(condition: u32, text: const u32[], index: usize) -> bool:
let base = condition & 7
var holds = true
if base == condition_final_sigma:
holds = not is_followed_by_cased_letter(text, index, 1) and is_followed_by_cased_letter(text, index, -1)
elif base == condition_after_soft_dotted:
holds = is_preceded_by_soft_dotted(text, index)
elif base == condition_more_above:
holds = is_followed_by_more_above(text, index)
elif base == condition_before_dot:
holds = is_followed_by_dot_above(text, index)
elif base == condition_after_i:
holds = is_preceded_by_i(text, index)
return not holds if (condition & condition_not) != 0 else holds
let conditional_mapping_digits: usize = 23
## Append the conditional mapping (field 0 lower, 1 title, 2 upper) of text[index] that
## applies in `locale`, and answer whether one did.
func append_conditional_case_mapping(text: const u32[], index: usize, locale: CaseLocale, field: usize, out: CodePointBuffer*) -> bool:
let code_point = text[index]
var entry: usize = 0
while entry < ucd_conditional_case_mappings.length:
if ucd_hex(ucd_conditional_case_mappings, entry, 6) == code_point:
let language = ucd_hex(ucd_conditional_case_mappings, entry + 6, 1)
let applies_to_locale = language == 0 or (language == 1 and locale == CaseLocale.lithuanian) or ((language == 2 or language == 3) and locale == CaseLocale.turkish)
if applies_to_locale and case_condition_holds(ucd_hex(ucd_conditional_case_mappings, entry + 7, 1), text, index):
let mapping = entry + 8 + field * 5
let length = ucd_hex(ucd_conditional_case_mappings, mapping + 4, 1)
if length == 0xf:
out.append(code_point)
else:
append_case_pool(ucd_hex(ucd_conditional_case_mappings, mapping, 4), length, out)
return true
entry += conditional_mapping_digits
return false
# mark: Full case mapping of one code point ======================================================
## ucase_toFullLower.
func append_full_lowercase(text: const u32[], index: usize, locale: CaseLocale, out: CodePointBuffer*):
if append_conditional_case_mapping(text, index, locale, 0, out):
return
append_full_case_mapping(text[index], case_full_lower, case_simple_lower, out)
## ucase_toFullUpper and ucase_toFullTitle.
func append_full_uppercase_or_titlecase(text: const u32[], index: usize, locale: CaseLocale, upper_not_title: bool, out: CodePointBuffer*):
let code_point = text[index]
if code_point == 0x587 and locale == CaseLocale.armenian:
# ICU-13416: ech-yiwn uppercases to ech+vew in Eastern Armenian.
out.append(0x535)
out.append(0x54e if upper_not_title else 0x57e)
return
if append_conditional_case_mapping(text, index, locale, 2 if upper_not_title else 1, out):
return
if upper_not_title:
append_full_case_mapping(code_point, case_full_upper, case_simple_upper, out)
else:
append_full_case_mapping(code_point, case_full_title, case_simple_title, out)
## ucase_toFullFolding; `turkic` is U_FOLD_CASE_EXCLUDE_SPECIAL_I.
func append_full_case_folding(code_point: u32, turkic: bool, out: CodePointBuffer*):
if turkic:
var entry: usize = 0
while entry < ucd_turkic_case_folding.length:
if ucd_hex(ucd_turkic_case_folding, entry, 6) == code_point:
append_case_pool(ucd_hex(ucd_turkic_case_folding, entry + 6, 4), ucd_hex(ucd_turkic_case_folding, entry + 10, 1), out)
return
entry += 11
append_full_case_mapping(code_point, case_full_fold, case_simple_fold, out)
# mark: Full case mapping of text ================================================================
## u_strToLower.
func lowercase_code_points(text: const u32[], locale: CaseLocale) -> CodePointBuffer:
var out = CodePointBuffer.create(text.length + 1)
for index in 0..<text.length:
append_full_lowercase(text, index, locale, &out)
return out
## u_strToUpper (the root locale's rules for Greek as well).
func uppercase_code_points(text: const u32[], locale: CaseLocale) -> CodePointBuffer:
var out = CodePointBuffer.create(text.length + 1)
for index in 0..<text.length:
append_full_uppercase_or_titlecase(text, index, locale, true, &out)
return out
## u_strFoldCase.
func case_fold_code_points(text: const u32[], turkic: bool) -> CodePointBuffer:
var out = CodePointBuffer.create(text.length + 1)
for code_point in text:
append_full_case_folding(code_point, turkic, &out)
return out
## ustrcase_isLNS: a letter, number, symbol or private use character; a modifier letter
## only when cased.
func is_lns(code_point: u32) -> bool:
let lns = (u_gc_l_mask | u_gc_n_mask | u_gc_s_mask | u_mask(17)) & ~u_mask(u_modifier_letter)
let gc = ucd_general_category(code_point)
return (u_mask(gc) & lns) != 0 or (gc == u_modifier_letter and ucd_has_binary_property(code_point, uchar_cased))
## ustrcase_internalToTitle with a word break iterator and options 0 or
## U_TITLECASE_NO_LOWERCASE (`no_lowercase`).
func titlecase_code_points(text: const u32[], locale: CaseLocale, no_lowercase: bool) -> CodePointBuffer:
var out = CodePointBuffer.create(text.length + 1)
var boundaries = CodePointBuffer.create(text.length + 2)
var statuses = CodePointBuffer.create(text.length + 2)
word_boundaries(text, &boundaries, &statuses)
var previous: usize = 0
for boundary in boundaries.view():
let index = (usize)boundary
# Segment [previous..index[ into: skipped characters (copied), the first letter
# (titlecased), the rest (lowercased).
if previous < index:
var title_start = previous
var title_limit = previous + 1
# Adjust the titlecasing index to the next letter, number, symbol or private use.
while not is_lns(text[title_limit - 1]):
title_start = title_limit
if title_limit == index:
break
title_limit += 1
for position in previous..<title_start:
out.append(text[position])
if title_start < title_limit:
let before_title = out.count
append_full_uppercase_or_titlecase(text, title_start, locale, false, &out)
# Special case Dutch IJ titlecasing.
if title_start + 1 < index and locale == CaseLocale.dutch and out.count == before_title + 1:
let titled = out.items[out.count - 1]
if titled == 0x49 or titled == 0xcd:
title_limit = maybe_title_dutch_ij(text, titled, title_limit, index, &out)
if title_limit < index:
if no_lowercase:
for position in title_limit..<index:
out.append(text[position])
else:
for position in title_limit..<index:
append_full_lowercase(text, position, locale, &out)
previous = index
return out
## ustrcase.cpp maybeTitleDutchIJ: after a titlecased I or Í, titlecase a following j (with
## a matching acute accent). Answers where lowercasing continues.
func maybe_title_dutch_ij(text: const u32[], titled: u32, start: usize, limit: usize, out: CodePointBuffer*) -> usize:
let acute: u32 = 0x301
var index = start
var with_acute = false
var unchanged1: usize = 0
var do_title_j = false
var unchanged2: usize = 0
var c2 = text[index]
index += 1
if titled == 0x49:
if c2 == acute:
with_acute = true
unchanged1 = 1
if index == limit:
return start
c2 = text[index]
index += 1
else:
with_acute = true
if c2 == 0x6a:
do_title_j = true
elif c2 == 0x4a:
unchanged1 += 1
else:
return start
if with_acute:
if index == limit or text[index] != acute:
return start
index += 1
if do_title_j:
unchanged2 = 1
else:
unchanged1 += 1
if index < limit and (ucd_general_category_mask(text[index]) & u_gc_m_mask) != 0:
return start
var position = start
for _unused in 0..<unchanged1:
out.append(text[position])
position += 1
if do_title_j:
out.append(0x4a)
position += 1
for _unused in 0..<unchanged2:
out.append(text[position])
position += 1
return index
# mark: Case closure ============================================================================
## The other code points of the case closure class of `code_point` (ucase_addCaseClosure,
## transitively, as UnicodeSet::closeOver(USET_CASE_INSENSITIVE) adds them).
func append_case_closure(code_point: u32, out: CodePointBuffer*):
var low: usize = 0
var high = ucd_case_closure.length // 12
while low < high:
let middle = low + (high - low) // 2
let key = ucd_hex(ucd_case_closure, middle * 12, 6)
if key < code_point:
low = middle + 1
elif key > code_point:
high = middle
else:
let entry = middle * 12
append_case_pool(ucd_hex(ucd_case_closure, entry + 6, 4), ucd_hex(ucd_case_closure, entry + 10, 2), out)
return
## Whether the case closure of `code_point` (the code point included) has a member for which
## `predicate(context, member)` holds.
func case_closure_any(code_point: u32, context: const void*, predicate: func(const void*, u32) -> bool) -> bool:
if predicate(context, code_point):
return true
var members = CodePointBuffer.create(4)
append_case_closure(code_point, &members)
for member in members.view():
if predicate(context, member):
return true
return false
# mark: Width ===================================================================================
let fullwidth_mapping_digits: usize = 13
## The Halfwidth-Fullwidth entry of `code_point`, or none.
func fullwidth_entry(code_point: u32) -> usize?:
var low: usize = 0
var high = ucd_fullwidth_mappings.length // fullwidth_mapping_digits
while low < high:
let middle = low + (high - low) // 2
let key = ucd_hex(ucd_fullwidth_mappings, middle * fullwidth_mapping_digits, 6)
if key < code_point:
low = middle + 1
elif key > code_point:
high = middle
else:
return middle * fullwidth_mapping_digits
return none
func is_halfwidth_hangul(code_point: u32) -> bool:
return code_point >= 0xffa0 and code_point <= 0xffdc
## The Halfwidth-Fullwidth transliterator: characters with a <wide> counterpart become it;
## runs of <narrow> characters are put in NFKC, so that a halfwidth katakana and its voiced
## sound mark become one character and halfwidth Hangul becomes
## conjoining jamo (NFKD, not composed).
func fullwidth_code_points(text: const u32[]) -> CodePointBuffer:
var out = CodePointBuffer.create(text.length + 1)
var index: usize = 0
while index < text.length:
let found = fullwidth_entry(text[index])
if found == none:
out.append(text[index])
index += 1
continue
let entry = found else break
if ucd_hex(ucd_fullwidth_mappings, entry + 12, 1) == 0:
out.append(ucd_hex(ucd_fullwidth_mappings, entry + 6, 6))
index += 1
continue
# Halfwidth Hangul becomes conjoining jamo, which ICU's rules do not compose.
if is_halfwidth_hangul(text[index]):
let single: u32[1] = [text[index]]
let jamo = normalize_code_points(single, NormalizationForm.nfkd)
out.append_all(jamo.view())
index += 1
continue
# A run of other narrow characters.
var run = CodePointBuffer.create(4)
while index < text.length and not is_halfwidth_hangul(text[index]):
let narrow = fullwidth_entry(text[index]) else break
if ucd_hex(ucd_fullwidth_mappings, narrow + 12, 1) == 0:
break
run.append(text[index])
index += 1
let composed = normalize_code_points(run.view(), NormalizationForm.nfkc)
out.append_all(composed.view())
return out