Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / text_codec / encoder_2.lucb

#==============================================================================================
#
#   encoder_2 - the Shift_JIS, EUC-KR, Big5, gb18030 and single-byte encoders, isomorphic encode
#
#   DESCRIPTION:
#       Port of Libraries/LibTextCodec/Encoder.cpp (lines 338-689) with the inline constructor of
#       Encoder.h's SingleByteEncoder: the index pointer lookups that skip or search parts of an
#       index, the encoders of Shift_JIS, EUC-KR, Big5, gb18030 (and GBK) and the single-byte
#       encodings, and Infra's isomorphic encode.
#
#==============================================================================================

# mark: Shift_JIS ==============================================================================

# Ported from Libraries/LibTextCodec/Encoder.cpp:338 TextCodec::code_point_jis0208_index_skipping_range.
func code_point_jis0208_index_skipping_range(code_point: u32, skip_from: u32, skip_to: u32) -> u32?:
    assert(skip_to >= skip_from)
    for i in 0..<s_jis0208_index.length:
        if i >= skip_from and i <= skip_to:
            continue
        if s_jis0208_index[i] == code_point:
            return (u32)i
    return none

# Ported from Libraries/LibTextCodec/Encoder.cpp:351 TextCodec::index_shift_jis_pointer.
# https://encoding.spec.whatwg.org/#index-shift_jis-pointer
func index_shift_jis_pointer(code_point: u32) -> u32?:
    # 1. Let index be index jis0208 excluding all entries whose pointer is in the range 8272 to 8835, inclusive.
    let pointer = code_point_jis0208_index_skipping_range(code_point, 8272, 8835)
    if pointer == none:
        return none

    # 2. Return the index pointer for code point in index.
    return pointer

## Ported from Libraries/LibTextCodec/Encoder.cpp:363 TextCodec::ShiftJISEncoder::process (override).
## https://encoding.spec.whatwg.org/#shift_jis-encoder
pub func shift_jis_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    # for (u32 item : input)
    let end = ak.utf8_view_end(&input)
    var iterator = ak.utf8_view_begin(&input)
    while iterator != end:
        var item = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        # 1. If code point is end-of-queue, return finished.

        # 2. If code point is an ASCII code point or U+0080, return a byte whose value is code point.
        if ak.is_ascii(item) or item == 0x0080:
            ak.function1_call(&on_byte, (u8)item)
            continue

        # 3. If code point is U+00A5, return byte 0x5C.
        if item == 0x00A5:
            ak.function1_call(&on_byte, (u8)0x5C)
            continue

        # 4. If code point is U+203E, return byte 0x7E.
        if item == 0x203E:
            ak.function1_call(&on_byte, (u8)0x7E)
            continue

        # 5. If code point is in the range U+FF61 to U+FF9F, inclusive, return a byte whose value is code point − 0xFF61 + 0xA1.
        if item >= 0xFF61 and item <= 0xFF9F:
            ak.function1_call(&on_byte, (u8)(item - 0xFF61 + 0xA1))
            continue

        # 6. If code point is U+2212, set it to U+FF0D.
        if item == 0x2212:
            item = 0xFF0D

        # 7. Let pointer be the index Shift_JIS pointer for code point.
        let pointer = index_shift_jis_pointer(item)

        # 8. If pointer is null, return error with code point.
        if pointer == none:
            ak.function1_call(&on_error, item)
            continue
        let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")

        # 9. Let lead be pointer / 188.
        let lead = index_pointer // 188

        # 10. Let lead offset be 0x81 if lead is less than 0x1F, otherwise 0xC1.
        var lead_offset: u32 = 0xC1
        if lead < 0x1F:
            lead_offset = 0x81

        # 11. Let trail be pointer % 188.
        let trail = index_pointer % 188

        # 12. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
        var offset: u32 = 0x41
        if trail < 0x3F:
            offset = 0x40

        # 13. Return two bytes whose values are lead + lead offset and trail + offset.
        ak.function1_call(&on_byte, (u8)(lead + lead_offset))
        ak.function1_call(&on_byte, (u8)(trail + offset))

# mark: EUC-KR =================================================================================

## Ported from Libraries/LibTextCodec/Encoder.cpp:430 TextCodec::EUCKREncoder::process (override).
## https://encoding.spec.whatwg.org/#euc-kr-encoder
pub func euckr_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    # for (u32 item : input)
    let end = ak.utf8_view_end(&input)
    var iterator = ak.utf8_view_begin(&input)
    while iterator != end:
        let item = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        # 1. If code point is end-of-queue, return finished.

        # 2. If code point is an ASCII code point, return a byte whose value is code point.
        if ak.is_ascii(item):
            ak.function1_call(&on_byte, (u8)item)
            continue

        # 3. Let pointer be the index pointer for code point in index EUC-KR.
        let pointer = code_point_euc_kr_index(item)

        # 4. If pointer is null, return error with code point.
        if pointer == none:
            ak.function1_call(&on_error, item)
            continue
        let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")

        # 5. Let lead be pointer / 190 + 0x81.
        let lead = index_pointer // 190 + 0x81

        # 6. Let trail be pointer % 190 + 0x41.
        let trail = index_pointer % 190 + 0x41

        # 7. Return two bytes whose values are lead and trail.
        ak.function1_call(&on_byte, (u8)lead)
        ak.function1_call(&on_byte, (u8)trail)

# mark: Big5 ===================================================================================

# Ported from Libraries/LibTextCodec/Encoder.cpp:465 TextCodec::index_big5_pointer.
# https://encoding.spec.whatwg.org/#index-big5-pointer
func index_big5_pointer(code_point: u32) -> u32?:
    # 1. Let index be index Big5 excluding all entries whose pointer is less than (0xA1 - 0x81) × 157.
    let start_index: u32 = (0xA1 - 0x81) * 157 - s_big5_index_first_pointer

    # 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345, return the last pointer
    #    corresponding to code point in index.
    let last_pointer_code_points: u32[6] = [0x2550, 0x255E, 0x2561, 0x256A, 0x5341, 0x5345]
    var contains = false  # Array<u32, 6> { ... }.contains_slow(code_point)
    for candidate in last_pointer_code_points:
        if candidate == code_point:
            contains = true
    if contains:
        var i: u32 = (u32)s_big5_index.length - 1
        while i >= start_index:
            if s_big5_index[i] == code_point:
                return s_big5_index_first_pointer + i
            i -= 1
        return none

    # 3. Return the index pointer for code point in index.
    var i: u32 = start_index
    while i < s_big5_index.length:
        if s_big5_index[i] == code_point:
            return s_big5_index_first_pointer + i
        i += 1
    return none

## Ported from Libraries/LibTextCodec/Encoder.cpp:491 TextCodec::Big5Encoder::process (override).
## https://encoding.spec.whatwg.org/#big5-encoder
pub func big5_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    # for (u32 item : input)
    let end = ak.utf8_view_end(&input)
    var iterator = ak.utf8_view_begin(&input)
    while iterator != end:
        let item = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        # 1. If code point is end-of-queue, return finished.

        # 2. If code point is an ASCII code point, return a byte whose value is code point.
        if ak.is_ascii(item):
            ak.function1_call(&on_byte, (u8)item)
            continue

        # 3. Let pointer be the index Big5 pointer for code point.
        let pointer = index_big5_pointer(item)

        # 4. If pointer is null, return error with code point.
        if pointer == none:
            ak.function1_call(&on_error, item)
            continue
        let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")

        # 5. Let lead be pointer / 157 + 0x81.
        let lead = index_pointer // 157 + 0x81

        # 6. Let trail be pointer % 157.
        let trail = index_pointer % 157

        # 7. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x62.
        var offset: u32 = 0x62
        if trail < 0x3f:
            offset = 0x40

        # 8. Return two bytes whose values are lead and trail + offset.
        ak.function1_call(&on_byte, (u8)lead)
        ak.function1_call(&on_byte, (u8)(trail + offset))

# mark: gb18030 ================================================================================

# index_gb18030_ranges_pointer's binary_search comparator:
# [](auto const code_point, auto const& entry) { return code_point - entry.code_point; } in u32, as int.
func gb18030_ranges_compare_code_point(context: void*?, code_point: const u32*, entry: const Gb18030RangeEntry*) -> i32:
    return (i32)(*code_point -% entry.code_point)

# Ported from Libraries/LibTextCodec/Encoder.cpp:531 TextCodec::index_gb18030_ranges_pointer.
# https://encoding.spec.whatwg.org/#index-gb18030-ranges-pointer
func index_gb18030_ranges_pointer(code_point: u32) -> u32:
    # 1. If code point is U+E7C7, return pointer 7457.
    if code_point == 0xe7c7:
        return 7457

    # 2. Let offset be the last code point in index gb18030 ranges that is less than
    #    or equal to code point and let pointer offset be its corresponding pointer.
    var last_index: usize = 0
    let comparator = ak.Function2[const u32*, const Gb18030RangeEntry*, i32](call = gb18030_ranges_compare_code_point, context = none)
    _ = ak.binary_search(s_gb18030_ranges, code_point, &last_index, comparator)
    let offset = s_gb18030_ranges[last_index].code_point
    let pointer_offset = s_gb18030_ranges[last_index].pointer

    # 3. Return a pointer whose value is pointer offset + code point − offset.
    return pointer_offset + code_point - offset

## Ported from Libraries/LibTextCodec/Encoder.cpp:550 TextCodec::GB18030Encoder::GB18030Encoder: a
## gb18030 encoder, or the GBK encoder when is_gbk is yes.
pub func gb18030_encoder_construct(this: Gb18030Encoder*, is_gbk: Gb18030EncoderIsGbk = Gb18030EncoderIsGbk.no):
    gb18030_encoder_init_fields(this)
    this.encoder_base.vtable = &gb18030_encoder_vtable.encoder_vtable  # the C++ vptr
    this.encoder_base.class_id = gb18030_encoder_class.first_id
    this.m_is_gbk = is_gbk

## Ported from Libraries/LibTextCodec/Encoder.cpp:556 TextCodec::GB18030Encoder::process (override).
## https://encoding.spec.whatwg.org/#gb18030-encoder
pub func gb18030_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    let this = (Gb18030Encoder*)base
    let gbk = this.m_is_gbk == Gb18030EncoderIsGbk.yes

    # for (u32 item : input)
    let end = ak.utf8_view_end(&input)
    var iterator = ak.utf8_view_begin(&input)
    while iterator != end:
        let item = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        # 1. If code point is end-of-queue, return finished.

        # 2. If code point is an ASCII code point, return a byte whose value is code point.
        if ak.is_ascii(item):
            ak.function1_call(&on_byte, (u8)item)
            continue

        # 3. If code point is U+E5E5, return error with code point.
        if item == 0xE5E5:
            ak.function1_call(&on_error, item)
            continue

        # 4. If is GBK is true and code point is U+20AC, return byte 0x80.
        if gbk and item == 0x20AC:
            ak.function1_call(&on_byte, (u8)0x80)
            continue

        # 5. Let pointer be the index pointer for code point in index gb18030.
        let pointer = code_point_gb18030_index(item)

        # 6. If pointer is non-null, then:
        if let index_pointer = pointer:
            # 1. Let lead be pointer / 190 + 0x81.
            let lead = index_pointer // 190 + 0x81

            # 2. Let trail be pointer % 190.
            let trail = index_pointer % 190

            # 3. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
            var offset: u32 = 0x41
            if trail < 0x3f:
                offset = 0x40

            # 4. Return two bytes whose values are lead and trail + offset.
            ak.function1_call(&on_byte, (u8)lead)
            ak.function1_call(&on_byte, (u8)(trail + offset))
            continue

        # 7. If is GBK is true, return error with code point.
        if gbk:
            ak.function1_call(&on_error, item)
            continue

        # 8. Set pointer to the index gb18030 ranges pointer for code point.
        var ranges_pointer = index_gb18030_ranges_pointer(item)

        # 9. Let byte1 be pointer / (10 × 126 × 10).
        let byte1 = ranges_pointer // (10 * 126 * 10)

        # 10. Set pointer to pointer % (10 × 126 × 10).
        ranges_pointer = ranges_pointer % (10 * 126 * 10)

        # 11. Let byte2 be pointer / (10 × 126).
        let byte2 = ranges_pointer // (10 * 126)

        # 12. Set pointer to pointer % (10 × 126).
        ranges_pointer = ranges_pointer % (10 * 126)

        # 13. Let byte3 be pointer / 10.
        let byte3 = ranges_pointer // 10

        # 14. Let byte4 be pointer % 10.
        let byte4 = ranges_pointer % 10

        # 15. Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30.
        ak.function1_call(&on_byte, (u8)(byte1 + 0x81))
        ak.function1_call(&on_byte, (u8)(byte2 + 0x30))
        ak.function1_call(&on_byte, (u8)(byte3 + 0x81))
        ak.function1_call(&on_byte, (u8)(byte4 + 0x30))

# mark: Single-byte encoder ====================================================================

# The donor instantiates SingleByteEncoder<ArrayType> only with ArrayType = u16 (its tables are
# Array<u16, 128>), and a Base generic body cannot compare a T with a u32, so the port's functions
# take the u16 instantiation.

## Ported from Libraries/LibTextCodec/Encoder.h:80 TextCodec::SingleByteEncoder::SingleByteEncoder<ArrayType>:
## a single-byte encoder over an index of the code points of bytes 0x80 to 0xFF.
pub func single_byte_encoder_construct(this: SingleByteEncoder[u16]*, translation_table: u16[128]):
    this.encoder_base.vtable = &single_byte_encoder_vtable.encoder_vtable  # the C++ vptr
    this.encoder_base.class_id = single_byte_encoder_class.first_id
    this.m_translation_table = translation_table

## Ported from Libraries/LibTextCodec/Encoder.cpp:642 TextCodec::SingleByteEncoder::process.
## https://encoding.spec.whatwg.org/#single-byte-encoder
pub func single_byte_encoder_process(this: SingleByteEncoder[u16]*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    # for (u32 const code_point : input)
    let end = ak.utf8_view_end(&input)
    var iterator = ak.utf8_view_begin(&input)
    while iterator != end:
        let code_point = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        if code_point < 0x80:
            # 2. If code point is an ASCII code point, return a byte whose value is code point.
            ak.function1_call(&on_byte, (u8)code_point)
        else:
            var pointer: u8? = none
            for i: u8 in 0..<128:
                if this.m_translation_table[i] == code_point:
                    # 3. Let pointer be the index pointer for code point in index single-byte.
                    pointer = i
                    break
            if let index_pointer = pointer:
                # 5. Return a byte whose value is pointer + 0x80.
                ak.function1_call(&on_byte, index_pointer + 0x80)
            else:
                # 4. If pointer is null, return error with code point.
                ak.function1_call(&on_error, code_point)
    # 1. If code point is end-of-queue, return finished.
    return

## The vtable slot of SingleByteEncoder<u16>::process (override): dispatches to
## single_byte_encoder_process.
pub func single_byte_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
    try single_byte_encoder_process((SingleByteEncoder[u16]*)base, input, on_byte, on_error)

# mark: Isomorphic encode ======================================================================

## Ported from Libraries/LibTextCodec/Encoder.cpp:671 TextCodec::isomorphic_encode: the bytes whose
## values are the input's code points (encoding as ISO-8859-1 / Latin-1).
## https://infra.spec.whatwg.org/#isomorphic-encode
pub func isomorphic_encode(input: ak.StringView) -> ak.ByteString:
    # To isomorphic encode an isomorphic string input: return a byte sequence whose length is equal to input’s code
    # point length and whose bytes have the same values as the values of input’s code points, in the same order.
    # NB: This is essentially spec-speak for "Encode as ISO-8859-1 / Latin-1".
    var builder: ak.StringBuilder
    ak.string_builder_construct_usize(&builder, ak.string_view_length(&input))

    # for (auto code_point : Utf8View { input })
    var view: ak.Utf8View
    ak.utf8_view_construct_string_view(&view, input)
    let end = ak.utf8_view_end(&view)
    var iterator = ak.utf8_view_begin(&view)
    while iterator != end:
        let code_point = ak.utf8_code_point_iterator_deref(&iterator)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

        # VERIFY(code_point <= 0xFF);
        if code_point > 0xFF:
            ak.dbgln("FIXME: Trying to isomorphic encode a string with code points > U+00FF.")

        ak.string_builder_append_u8(&builder, (u8)code_point)

    return ak.string_builder_to_byte_string(&builder)