Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / ak / unicode_utils.lucb

#==============================================================================================
#
#   unicode_utils - UTF-8 and UTF-16 encoding of single code points
#
#   DESCRIPTION:
#       Port of AK/UnicodeUtils.h (lines 1-212). The callback templates take a pointer to a
#       struct conforming to the callback interfaces of text_support.
#
#==============================================================================================

# mark: UTF-8 ===================================================================================

## Ported from AK/UnicodeUtils.h:14 AK::UnicodeUtils::bytes_to_store_code_point_in_utf8.
pub func unicode_utils_bytes_to_store_code_point_in_utf8(code_point: u32) -> i32:
    if code_point <= 0x7f:
        return 1
    if code_point <= 0x7ff:
        return 2
    if code_point <= 0xffff:
        return 3
    if code_point <= 0x10ffff:
        return 4
    return 0

## Ported from AK/UnicodeUtils.h:28 AK::UnicodeUtils::code_point_to_utf8.
pub func unicode_utils_code_point_to_utf8[TCallback: Utf8ByteCallback](code_point: u32, callback: TCallback*) -> i32:
    if code_point <= 0x7f:
        callback.call((u8)code_point)
        return 1
    elif code_point <= 0x07ff:
        callback.call((u8)(((code_point >> 6) & 0x1f) | 0xc0))
        callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 2
    elif code_point <= 0xffff:
        callback.call((u8)(((code_point >> 12) & 0x0f) | 0xe0))
        callback.call((u8)(((code_point >> 6) & 0x3f) | 0x80))
        callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 3
    elif code_point <= 0x10ffff:
        callback.call((u8)(((code_point >> 18) & 0x07) | 0xf0))
        callback.call((u8)(((code_point >> 12) & 0x3f) | 0x80))
        callback.call((u8)(((code_point >> 6) & 0x3f) | 0x80))
        callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 4
    return -1

## Ported from AK/UnicodeUtils.h:52 AK::UnicodeUtils::try_code_point_to_utf8.
pub func unicode_utils_try_code_point_to_utf8[TCallback: FallibleUtf8ByteCallback](code_point: u32, callback: TCallback*) -> i32!:
    if code_point <= 0x7f:
        try callback.call((u8)code_point)
        return 1
    if code_point <= 0x07ff:
        try callback.call((u8)(((code_point >> 6) & 0x1f) | 0xc0))
        try callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 2
    if code_point <= 0xffff:
        try callback.call((u8)(((code_point >> 12) & 0x0f) | 0xe0))
        try callback.call((u8)(((code_point >> 6) & 0x3f) | 0x80))
        try callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 3
    if code_point <= 0x10ffff:
        try callback.call((u8)(((code_point >> 18) & 0x07) | 0xf0))
        try callback.call((u8)(((code_point >> 12) & 0x3f) | 0x80))
        try callback.call((u8)(((code_point >> 6) & 0x3f) | 0x80))
        try callback.call((u8)(((code_point >> 0) & 0x3f) | 0x80))
        return 4
    return -1

## A code_point_to_utf8 callback that writes the bytes into a four-byte array (the C++'s
## `Array<u8, 4> code_point_as_utf8; code_point_to_utf8(cp, [&](auto byte) { ... })`).
pub struct CodePointAsUtf8: Utf8ByteCallback:
    pub var bytes: u8[4]
    pub var length: usize

    pub func call(byte: u8):
        self.bytes[self.length] = byte
        self.length += 1

# mark: UTF-16 ==================================================================================

## Ported from AK/UnicodeUtils.h:93 AK::UnicodeUtils::code_unit_length_for_code_point.
pub func unicode_utils_code_unit_length_for_code_point(code_point: u32) -> usize:
    return 1 if code_point < unicode_utils_first_supplementary_plane_code_point else 2

## Ported from AK/UnicodeUtils.h:98 AK::UnicodeUtils::is_utf16_high_surrogate.
pub func unicode_utils_is_utf16_high_surrogate(code_unit: u16) -> bool:
    return (code_unit >= unicode_utils_high_surrogate_min) and (code_unit <= unicode_utils_high_surrogate_max)

## Ported from AK/UnicodeUtils.h:103 AK::UnicodeUtils::is_utf16_low_surrogate.
pub func unicode_utils_is_utf16_low_surrogate(code_unit: u16) -> bool:
    return (code_unit >= unicode_utils_low_surrogate_min) and (code_unit <= unicode_utils_low_surrogate_max)

## Ported from AK/UnicodeUtils.h:108 AK::UnicodeUtils::decode_utf16_surrogate_pair.
pub func unicode_utils_decode_utf16_surrogate_pair(high_surrogate: u16, low_surrogate: u16) -> u32:
    assert(unicode_utils_is_utf16_high_surrogate(high_surrogate))
    assert(unicode_utils_is_utf16_low_surrogate(low_surrogate))

    return (((u32)high_surrogate - unicode_utils_high_surrogate_min) << 10) + ((u32)low_surrogate - unicode_utils_low_surrogate_min) + unicode_utils_first_supplementary_plane_code_point

## Ported from AK/UnicodeUtils.h:117 AK::UnicodeUtils::code_point_to_utf16.
pub func unicode_utils_code_point_to_utf16[TCallback: Utf16CodeUnitCallback](code_point: u32, callback: TCallback*) -> usize:
    if code_point < unicode_utils_first_supplementary_plane_code_point:
        callback.call((u16)code_point)
        return 1

    if code_point <= 0x10ffff:
        let supplementary = code_point - unicode_utils_first_supplementary_plane_code_point

        var code_unit = (u16)((u32)unicode_utils_high_surrogate_min | (supplementary >> 10))
        callback.call(code_unit)

        code_unit = (u16)((u32)unicode_utils_low_surrogate_min | (supplementary & 0x3ff))
        callback.call(code_unit)

        return 2

    trap("VERIFY_NOT_REACHED")

## Ported from AK/UnicodeUtils.h:139 AK::UnicodeUtils::try_code_point_to_utf16.
pub func unicode_utils_try_code_point_to_utf16[TCallback: FallibleUtf16CodeUnitCallback](code_point: u32, callback: TCallback*) -> usize!:
    if code_point < unicode_utils_first_supplementary_plane_code_point:
        try callback.call((u16)code_point)
        return 1

    if code_point <= 0x10ffff:
        let supplementary = code_point - unicode_utils_first_supplementary_plane_code_point

        var code_unit = (u16)((u32)unicode_utils_high_surrogate_min | (supplementary >> 10))
        try callback.call(code_unit)

        code_unit = (u16)((u32)unicode_utils_low_surrogate_min | (supplementary & 0x3ff))
        try callback.call(code_unit)

        return 2

    trap("VERIFY_NOT_REACHED")

## Ported from AK/UnicodeUtils.h:165 AK::UnicodeUtils::maximum_utf8_length_from_utf16.
## Computes the maximum number of UTF-8 bytes needed to store a given UTF-16 string, accounting
## for unmatched UTF-16 surrogates. This overcounts by at most 33%; 2 bytes for every valid
## UTF-16 code point between U+100000 and U+10FFFF.
func unicode_utils_maximum_utf8_length_from_utf16(code_units: const u16[]) -> usize:
    # # UTF-16 code unit -> actual length added.
    # 0x0000 - 0x007f = 1
    # 0x0080 - 0x07ff = 2
    # 0x0800 - 0xd7ff = 3
    # 0xd800 - 0xdbff = 3
    #   ^ If the next code unit is 0xdc00 - 0xdfff, they will combined sum to 6, which is greater
    #   than the 4 required. Otherwise, 3 bytes are needed to encode U+D800 - U+DBFF.
    # 0xdc00 - 0xdfff = 3
    #   ^ If the previous code unit was, 0xd800 - 0xdbff, this will ensure that the combined sum
    #   is greater than 4. Otherwise, 3 bytes are needed to encode U+DC00 - U+DFFF.
    # 0xe000 - 0xffff = 3
    var maximum_utf8_length: usize = 0

    for code_unit in code_units:
        maximum_utf8_length += 1
        if code_unit > 0x007f:
            maximum_utf8_length += 1
        if code_unit > 0x07ff:
            maximum_utf8_length += 1

    return maximum_utf8_length