Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / text_codec / decoder_3.lucb

#==============================================================================================
#
#   decoder_3 - the Unicode, Latin-1, PDFDocEncoding, x-user-defined and single-byte decoders
#
#   DESCRIPTION:
#       Port of Libraries/LibTextCodec/Decoder.cpp (lines 692-841) with the inline functions of
#       Decoder.h they override: UTF-8, UTF-16BE/LE (which decode through AK::String), Latin-1,
#       PDFDocEncoding, x-user-defined, and the single-byte decoder over an index of 128 code
#       points.
#
#==============================================================================================

# mark: UTF-8 ==================================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:692 TextCodec::UTF8Decoder::process (override).
pub func utf8_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    # for (auto c : Utf8View(input))
    var view: ak.Utf8View
    ak.utf8_view_construct_string_view(&view, input)
    let end = ak.utf8_view_end(&view)
    var iterator = ak.utf8_view_begin(&view)
    while iterator != end:
        let c = ak.utf8_code_point_iterator_deref(&iterator)
        ak.function1_call(&on_code_point, c)
        _ = ak.utf8_code_point_iterator_inc(&iterator)

## Ported from Libraries/LibTextCodec/Decoder.cpp:700 TextCodec::UTF8Decoder::validate (override).
pub func utf8_decoder_validate_impl(base: Decoder*, input: ak.StringView) -> bool:
    var view: ak.Utf8View
    ak.utf8_view_construct_string_view(&view, input)
    return ak.utf8_view_validate_allow_lonely_surrogates(&view)

## Ported from Libraries/LibTextCodec/Decoder.cpp:705 TextCodec::UTF8Decoder::to_utf8 (override).
pub func utf8_decoder_to_utf8_impl(base: Decoder*, input: ak.StringView) -> ak.String!:
    return ak.string_from_utf8_with_replacement_character(input)

# mark: UTF-16 =================================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:710 TextCodec::UTF16BEDecoder::validate (override).
pub func utf16_be_decoder_validate_impl(base: Decoder*, input: ak.StringView) -> bool:
    return ak.validate_utf16_be(ak.string_view_bytes(&input))

## Ported from Libraries/LibTextCodec/Decoder.cpp:715 TextCodec::UTF16BEDecoder::to_utf8 (override).
pub func utf16_be_decoder_to_utf8_impl(base: Decoder*, input: ak.StringView) -> ak.String!:
    var remaining_input = input  # C++ reassigns the `input` parameter

    # Discard the BOM
    let bytes = ak.string_view_bytes(&input)
    if bytes.length >= 2 and bytes[0] == 0xFE and bytes[1] == 0xFF:
        remaining_input = ak.string_view_substring_view(&input, 2)

    return try ak.string_from_utf16_be_with_replacement_character(ak.string_view_bytes(&remaining_input))

## Ported from Libraries/LibTextCodec/Decoder.h:50 TextCodec::UTF16BEDecoder::process (override):
## never called, to_utf8 and validate do not go through it.
pub func utf16_be_decoder_process_impl(base: Decoder*, arg0: ak.StringView, arg1: ak.Function1[u32, unit]) -> !:
    trap("VERIFY_NOT_REACHED")

## Ported from Libraries/LibTextCodec/Decoder.cpp:724 TextCodec::UTF16LEDecoder::validate (override).
pub func utf16_le_decoder_validate_impl(base: Decoder*, input: ak.StringView) -> bool:
    return ak.validate_utf16_le(ak.string_view_bytes(&input))

## Ported from Libraries/LibTextCodec/Decoder.cpp:729 TextCodec::UTF16LEDecoder::to_utf8 (override).
pub func utf16_le_decoder_to_utf8_impl(base: Decoder*, input: ak.StringView) -> ak.String!:
    var remaining_input = input  # C++ reassigns the `input` parameter

    # Discard the BOM
    let bytes = ak.string_view_bytes(&input)
    if bytes.length >= 2 and bytes[0] == 0xFF and bytes[1] == 0xFE:
        remaining_input = ak.string_view_substring_view(&input, 2)

    return try ak.string_from_utf16_le_with_replacement_character(ak.string_view_bytes(&remaining_input))

## Ported from Libraries/LibTextCodec/Decoder.h:60 TextCodec::UTF16LEDecoder::process (override):
## never called, to_utf8 and validate do not go through it.
pub func utf16_le_decoder_process_impl(base: Decoder*, arg0: ak.StringView, arg1: ak.Function1[u32, unit]) -> !:
    trap("VERIFY_NOT_REACHED")

# mark: Latin-1, PDFDocEncoding, x-user-defined ==============================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:738 TextCodec::Latin1Decoder::process (override).
pub func latin1_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    for ch in ak.string_view_bytes(&input):
        # Latin1 is the same as the first 256 Unicode code_points, so no mapping is needed, just utf-8 encoding.
        ak.function1_call(&on_code_point, (u32)ch)

## Ported from Libraries/LibTextCodec/Decoder.h:80 TextCodec::Latin1Decoder::validate (override).
pub func latin1_decoder_validate_impl(base: Decoder*, arg0: ak.StringView) -> bool:
    return true

# PDFDocEncodingDecoder::process's static translation table.
# PDF 1.7 spec, Appendix D.2 "PDFDocEncoding Character Set"
# Character codes 0-8, 11-12, 14-23, 127, 159, 173 are not defined per spec.
let pdf_doc_encoding_translation_table: u32[256] = [
    0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
    0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
    0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0017, 0x0017,
    0x02D8, 0x02C7, 0x02C6, 0x02D9, 0x02DD, 0x02DB, 0x02DA, 0x02DC,
    0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
    0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
    0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
    0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
    0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
    0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
    0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
    0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
    0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
    0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
    0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0xFFFC,
    0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044,
    0x2039, 0x203A, 0x2212, 0x2030, 0x201E, 0x201C, 0x201D, 0x2018,
    0x2019, 0x201A, 0x2122, 0xFB01, 0xFB02, 0x0141, 0x0152, 0x0160,
    0x0178, 0x017D, 0x0131, 0x0142, 0x0153, 0x0161, 0x017E, 0xFFFC,
    0x20AC, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
    0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0xFFFC, 0x00AE, 0x00AF,
    0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
    0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
    0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
    0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
    0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
    0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
    0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
    0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
    0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
    0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF,
]

## Ported from Libraries/LibTextCodec/Decoder.cpp:748 TextCodec::PDFDocEncodingDecoder::process (override).
pub func pdf_doc_encoding_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    for ch in ak.string_view_bytes(&input):
        ak.function1_call(&on_code_point, pdf_doc_encoding_translation_table[ch])

## Ported from Libraries/LibTextCodec/Decoder.h:86 TextCodec::PDFDocEncodingDecoder::validate (override).
pub func pdf_doc_encoding_decoder_validate_impl(base: Decoder*, arg0: ak.StringView) -> bool:
    return true

# XUserDefinedDecoder::process's lambda convert_x_user_defined_to_utf8.
func convert_x_user_defined_to_utf8(ch: u8) -> u32:
    # 2. If byte is an ASCII byte, return a code point whose value is byte.
    # https://infra.spec.whatwg.org/#ascii-byte
    # An ASCII byte is a byte in the range 0x00 (NUL) to 0x7F (DEL), inclusive.
    # NOTE: This doesn't check for ch >= 0x00, as that would always be true due to being unsigned.
    if ch <= 0x7f:
        return ch

    # 3. Return a code point whose value is 0xF780 + byte − 0x80.
    return 0xF780 + (u32)ch - 0x80

## Ported from Libraries/LibTextCodec/Decoder.cpp:796 TextCodec::XUserDefinedDecoder::process (override).
## https://encoding.spec.whatwg.org/#x-user-defined-decoder
pub func x_user_defined_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    for ch in ak.string_view_bytes(&input):
        ak.function1_call(&on_code_point, convert_x_user_defined_to_utf8(ch))

    # 1. If byte is end-of-queue, return finished.
    return

## Ported from Libraries/LibTextCodec/Decoder.h:92 TextCodec::XUserDefinedDecoder::validate (override).
pub func x_user_defined_decoder_validate_impl(base: Decoder*, arg0: ak.StringView) -> bool:
    return true

# mark: Single-byte decoder ====================================================================

# The donor instantiates SingleByteDecoder<ArrayType> only with ArrayType = u16 (its tables are
# Array<u16, 128>), and a Base generic body cannot convert a T to u32, so the port's functions take
# the u16 instantiation.

## Ported from Libraries/LibTextCodec/Decoder.h:66 TextCodec::SingleByteDecoder::SingleByteDecoder<ArrayType>:
## a single-byte decoder over an index of the code points of bytes 0x80 to 0xFF.
pub func single_byte_decoder_construct(this: SingleByteDecoder[u16]*, translation_table: u16[128]):
    this.decoder_base.vtable = &single_byte_decoder_vtable.decoder_vtable  # the C++ vptr
    this.decoder_base.class_id = single_byte_decoder_class.first_id
    this.m_translation_table = translation_table

## Ported from Libraries/LibTextCodec/Decoder.cpp:821 TextCodec::SingleByteDecoder::process.
## https://encoding.spec.whatwg.org/#single-byte-decoder
pub func single_byte_decoder_process(this: SingleByteDecoder[u16]*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    for byte in ak.string_view_bytes(&input):
        if byte < 0x80:
            # 2. If byte is an ASCII byte, return a code point whose value is byte.
            ak.function1_call(&on_code_point, (u32)byte)
        else:
            # 3. Let code point be the index code point for byte − 0x80 in index single-byte.
            let code_point = this.m_translation_table[byte - 0x80]

            # 4. If code point is null, return error.
            # NOTE: Error is communicated with 0xFFFD

            # 5. Return a code point whose value is code point.
            ak.function1_call(&on_code_point, (u32)code_point)
    # 1. If byte is end-of-queue, return finished.
    return

## The vtable slot of SingleByteDecoder<u16>::process (override): dispatches to
## single_byte_decoder_process.
pub func single_byte_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    try single_byte_decoder_process((SingleByteDecoder[u16]*)base, input, on_code_point)