dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / web_unicode / normalize.lucb
#==============================================================================================
#
# normalize - Unicode::normalize: the Unicode normalization forms
#
# DESCRIPTION:
# Port of LibUnicode/Normalize.cpp (lines 1-79). ICU's Normalizer2 is luce-std's
# unicode.normalize (Unicode 17, the version of ICU 78), as DESIGN.md ยง7.1 decides.
#
#==============================================================================================
## Ported from Libraries/LibUnicode/Normalize.cpp:16 Unicode::normalization_form_from_string.
pub func normalization_form_from_string(form: ak.StringView) -> NormalizationForm:
if view_equals(form, "NFD"):
return NormalizationForm.nfd
if view_equals(form, "NFC"):
return NormalizationForm.nfc
if view_equals(form, "NFKD"):
return NormalizationForm.nfkd
if view_equals(form, "NFKC"):
return NormalizationForm.nfkc
trap("VERIFY_NOT_REACHED")
## Ported from Libraries/LibUnicode/Normalize.cpp:29 Unicode::normalization_form_to_string.
pub func normalization_form_to_string(form: NormalizationForm) -> ak.StringView:
match form:
.nfd: return view_of("NFD")
.nfc: return view_of("NFC")
.nfkd: return view_of("NFKD")
.nfkc: return view_of("NFKC")
_: trap("VERIFY_NOT_REACHED")
## Ported from Libraries/LibUnicode/Normalize.cpp:44 Unicode::normalize.
pub func normalize(string: ak.StringView, form: NormalizationForm) -> ak.String:
# A failure of ICU (here: text that is not UTF-8) returns the text unchanged.
if let normalized = normalize_utf8(view_bytes(string), form):
defer normalized.release()
return ak.string_from_utf8(view_of(normalized.value)) else trap("MUST: StringBuilder::to_string")
return ak.string_from_utf8(string) else trap("MUST: String::from_utf8")
## Normalizer2::normalizeUTF8: `bytes` in `form`, or none when they are not UTF-8. The caller
## releases the result.
func normalize_utf8(bytes: const u8[], form: NormalizationForm) -> interop.Owned[str]?:
let text = str(bytes) else return none
let unicode_form = match form:
.nfd => unicode.NormalizationForm.nfd
.nfc => unicode.NormalizationForm.nfc
.nfkd => unicode.NormalizationForm.nfkd
.nfkc => unicode.NormalizationForm.nfkc
_ => trap("VERIFY_NOT_REACHED")
let normalized = unicode.normalize(text, unicode_form) else return none
return normalized
## The code points of `code_points` in `form` (lone surrogates become U+FFFD).
func normalize_code_points(code_points: const u32[], form: NormalizationForm) -> CodePointBuffer:
let bytes = encode_utf8_all(code_points)
var out = CodePointBuffer.create(code_points.length + 1)
if let normalized = normalize_utf8(bytes.view(), form):
defer normalized.release()
decode_utf8(normalized.value.bytes, &out, none)
else:
out.append_all(code_points)
return out