Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / text_codec / decoder_5.lucb

#==============================================================================================
#
#   decoder_5 - the ISO-2022-JP, Shift_JIS, EUC-KR and replacement decoders, isomorphic decode
#
#   DESCRIPTION:
#       Port of Libraries/LibTextCodec/Decoder.cpp (lines 1224-1653) with ReplacementDecoder's
#       inline validate from Decoder.h: the decoders of ISO-2022-JP (a state machine over escape
#       sequences), Shift_JIS, EUC-KR and replacement, and Infra's isomorphic decode.
#
#==============================================================================================

# mark: ISO-2022-JP ============================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:1235 TextCodec::ISO2022JPDecoder::process (override).
## https://encoding.spec.whatwg.org/#iso-2022-jp-decoder
pub func iso2022_jp_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    # ISO-2022-JP’s decoder has an associated ISO-2022-JP decoder state (initially ASCII), ISO-2022-JP decoder output state (initially ASCII), ISO-2022-JP lead (initially 0x00), and ISO-2022-JP output (initially false).
    var decoder_state = Iso2022JpState.ascii
    var output_state = Iso2022JpState.ascii
    var iso2022_jp_lead: u8 = 0x00
    var iso2022_jp_output = false

    var index: usize = 0
    while true:
        var byte: u8? = none
        if index < ak.string_view_length(&input):
            byte = ak.string_view_at(&input, index)
            index += 1
        let byte_value = byte else 0x00  # byte.value(), read only where byte has a value

        # ISO-2022-JP’s decoder’s handler, given ioQueue and byte, runs these steps, switching on ISO-2022-JP decoder state:
        match decoder_state:
            .ascii:
                # Based on byte:
                # 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
                if byte == 0x1B:
                    decoder_state = Iso2022JpState.escape_start
                    continue

                # 0x00 to 0x7F, excluding 0x0E, 0x0F, and 0x1B: Set ISO-2022-JP output to false and return a code point whose value is byte.
                if byte != none and byte_value <= 0x7F and byte != 0x0E and byte != 0x0F:
                    iso2022_jp_output = false
                    ak.function1_call(&on_code_point, (u32)byte_value)
                    continue

                # end-of-queue: Return finished.
                if byte == none:
                    return

                # Otherwise: Set ISO-2022-JP output to false and return error.
                iso2022_jp_output = false
                ak.function1_call(&on_code_point, replacement_code_point)
            .roman:
                # Based on byte:
                # 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
                if byte == 0x1B:
                    decoder_state = Iso2022JpState.escape_start
                    continue

                # 0x5C: Set ISO-2022-JP output to false and return code point U+00A5.
                if byte == 0x5C:
                    iso2022_jp_output = false
                    ak.function1_call(&on_code_point, 0x00A5)
                    continue

                # 0x7E: Set ISO-2022-JP output to false and return code point U+203E.
                if byte == 0x7E:
                    iso2022_jp_output = false
                    ak.function1_call(&on_code_point, 0x203E)
                    continue

                # 0x00 to 0x7F, excluding 0x0E, 0x0F, 0x1B, 0x5C, and 0x7E: Set ISO-2022-JP output to false and return a code point whose value is byte.
                if byte != none and byte_value <= 0x7F and byte != 0x0E and byte != 0x0F:
                    iso2022_jp_output = false
                    ak.function1_call(&on_code_point, (u32)byte_value)
                    continue

                # end-of-queue: Return finished.
                if byte == none:
                    return

                # Otherwise: Set ISO-2022-JP output to false and return error.
                iso2022_jp_output = false
                ak.function1_call(&on_code_point, replacement_code_point)
            .katakana:
                # Based on byte:
                # 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
                if byte == 0x1B:
                    decoder_state = Iso2022JpState.escape_start
                    continue

                # 0x21 to 0x5F: Set ISO-2022-JP output to false and return a code point whose value is 0xFF61 − 0x21 + byte.
                if byte != none and byte_value >= 0x21 and byte_value <= 0x5F:
                    iso2022_jp_output = false
                    ak.function1_call(&on_code_point, 0xFF61 - 0x21 + (u32)byte_value)
                    continue

                # end-of-queue: Return finished.
                if byte == none:
                    return

                # Otherwise: Set ISO-2022-JP output to false and return error.
                iso2022_jp_output = false
                ak.function1_call(&on_code_point, replacement_code_point)
            .lead_byte:
                # Based on byte:
                # 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
                if byte == 0x1B:
                    decoder_state = Iso2022JpState.escape_start
                    continue

                # 0x21 to 0x7E: Set ISO-2022-JP output to false, ISO-2022-JP lead to byte, ISO-2022-JP decoder state to trail byte, and return continue.
                if byte != none and byte_value >= 0x21 and byte_value <= 0x7E:
                    iso2022_jp_output = false
                    iso2022_jp_lead = byte_value
                    decoder_state = Iso2022JpState.trail_byte
                    continue

                # end-of-queue: Return finished.
                if byte == none:
                    return

                # Otherwise: Set ISO-2022-JP output to false and return error.
                iso2022_jp_output = false
                ak.function1_call(&on_code_point, replacement_code_point)
            .trail_byte:
                # Based on byte:

                # 0x1B: Set ISO-2022-JP decoder state to escape start and return error.
                if byte == 0x1B:
                    decoder_state = Iso2022JpState.escape_start
                    ak.function1_call(&on_code_point, replacement_code_point)
                    continue

                # 0x21 to 0x7E:
                if byte != none and byte_value >= 0x21 and byte_value <= 0x7E:
                    # 1. Set the ISO-2022-JP decoder state to lead byte.
                    decoder_state = Iso2022JpState.lead_byte

                    # 2. Let pointer be (ISO-2022-JP lead − 0x21) × 94 + byte − 0x21.
                    let pointer: u32 = ((u32)iso2022_jp_lead - 0x21) * 94 + (u32)byte_value - 0x21

                    # 3. Let code point be the index code point for pointer in index jis0208.
                    let code_point = index_jis0208_code_point(pointer)

                    # 4. If code point is null, return error.
                    if code_point == none:
                        ak.function1_call(&on_code_point, replacement_code_point)
                        continue

                    # 5. Return a code point whose value is code point.
                    ak.function1_call(&on_code_point, code_point else trap("VERIFY: Optional::value() of an empty Optional"))
                    continue

                # end-of-queue: Set the ISO-2022-JP decoder state to lead byte and return error.
                if byte == none:
                    decoder_state = Iso2022JpState.lead_byte
                    ak.function1_call(&on_code_point, replacement_code_point)
                    continue

                # Otherwise: Set ISO-2022-JP decoder state to lead byte and return error.
                decoder_state = Iso2022JpState.lead_byte
                ak.function1_call(&on_code_point, replacement_code_point)
            .escape_start:
                # 1. If byte is either 0x24 or 0x28, set ISO-2022-JP lead to byte, ISO-2022-JP decoder state to escape, and return continue.
                if byte == 0x24 or byte == 0x28:
                    iso2022_jp_lead = byte_value
                    decoder_state = Iso2022JpState.escape
                    continue

                # 2. If byte is not end-of-queue, then restore byte to ioQueue.
                if byte != none:
                    index -= 1

                # 3. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state, and return error.
                iso2022_jp_output = false
                decoder_state = output_state
                ak.function1_call(&on_code_point, replacement_code_point)
            .escape:
                # 1. Let lead be ISO-2022-JP lead and set ISO-2022-JP lead to 0x00.
                let lead = iso2022_jp_lead
                iso2022_jp_lead = 0x00

                # 2. Let state be null.
                var state: Iso2022JpState? = none

                # 3. If lead is 0x28 and byte is 0x42, set state to ASCII.
                if lead == 0x28 and byte == 0x42:
                    state = Iso2022JpState.ascii

                # 4. If lead is 0x28 and byte is 0x4A, set state to Roman.
                if lead == 0x28 and byte == 0x4A:
                    state = Iso2022JpState.roman

                # 5. If lead is 0x28 and byte is 0x49, set state to katakana.
                if lead == 0x28 and byte == 0x49:
                    state = Iso2022JpState.katakana

                # 6. If lead is 0x24 and byte is either 0x40 or 0x42, set state to lead byte.
                if lead == 0x24 and (byte == 0x40 or byte == 0x42):
                    state = Iso2022JpState.lead_byte

                # 7. If state is non-null, then:
                if let new_state = state:
                    # 1. Set ISO-2022-JP decoder state and ISO-2022-JP decoder output state to state.
                    decoder_state = new_state
                    output_state = new_state

                    # 2. Let output be the value of ISO-2022-JP output.
                    let output = iso2022_jp_output

                    # 3. Set ISO-2022-JP output to true.
                    iso2022_jp_output = true

                    # 4. Return continue, if output is false, and error otherwise.
                    if output:
                        ak.function1_call(&on_code_point, replacement_code_point)
                    continue

                # 8. If byte is end-of-queue, then restore lead to ioQueue; otherwise, restore « lead, byte » to ioQueue.
                index -= 2 if byte != none else 1

                # 9. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state and return error.
                iso2022_jp_output = false
                decoder_state = output_state
                ak.function1_call(&on_code_point, replacement_code_point)
            _:
                trap("VERIFY_NOT_REACHED: ISO2022JPState")

# mark: Shift_JIS ==============================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:1474 TextCodec::ShiftJISDecoder::process (override).
## https://encoding.spec.whatwg.org/#shift_jis-decoder
pub func shift_jis_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    # Shift_JIS’s decoder has an associated Shift_JIS lead (initially 0x00).
    var shift_jis_lead: u8 = 0x00

    # Shift_JIS’s decoder’s handler, given ioQueue and byte, runs these steps:
    var index: usize = 0
    while true:
        # 1. If byte is end-of-queue and Shift_JIS lead is not 0x00, set Shift_JIS lead to 0x00 and return error.
        if index >= ak.string_view_length(&input) and shift_jis_lead != 0x00:
            shift_jis_lead = 0x00
            ak.function1_call(&on_code_point, replacement_code_point)
            continue

        # 2. If byte is end-of-queue and Shift_JIS lead is 0x00, return finished.
        if index >= ak.string_view_length(&input) and shift_jis_lead == 0x00:
            return

        let byte = ak.string_view_at(&input, index)
        index += 1

        # 3. If Shift_JIS lead is not 0x00, let lead be Shift_JIS lead, let pointer be null, set Shift_JIS lead to 0x00, and then:
        if shift_jis_lead != 0x00:
            let lead = shift_jis_lead
            var pointer: u32? = none
            shift_jis_lead = 0x00

            # 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
            let offset: u8 = 0x40 if byte < 0x7F else 0x41

            # 2. Let lead offset be 0x81 if lead is less than 0xA0, otherwise 0xC1.
            let lead_offset: u8 = 0x81 if lead < 0xA0 else 0xC1

            # 3. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFC, inclusive, set pointer to (lead − lead offset) × 188 + byte − offset.
            if (byte >= 0x40 and byte <= 0x7E) or (byte >= 0x80 and byte <= 0xFC):
                pointer = (u32)(lead - lead_offset) * 188 + (u32)byte - (u32)offset

            # 4. If pointer is in the range 8836 to 10715, inclusive, return a code point whose value is 0xE000 − 8836 + pointer.
            if let p = pointer:
                if p >= 8836 and p <= 10715:
                    ak.function1_call(&on_code_point, 0xE000 - 8836 + p)
                    continue

            # 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index jis0208.
            var code_point: u32? = none
            if let p = pointer:
                code_point = index_jis0208_code_point(p)

            # 6. If code point is non-null, return a code point whose value is code point.
            if let value = code_point:
                ak.function1_call(&on_code_point, value)
                continue

            # 7. If byte is an ASCII byte, restore byte to ioQueue.
            if byte <= 0x7F:
                index -= 1

            # 8. Return error.
            ak.function1_call(&on_code_point, replacement_code_point)
            continue

        # 4. If byte is an ASCII byte or 0x80, return a code point whose value is byte.
        if byte <= 0x80:
            ak.function1_call(&on_code_point, (u32)byte)
            continue

        # 5. If byte is in the range 0xA1 to 0xDF, inclusive, return a code point whose value is 0xFF61 − 0xA1 + byte.
        if byte >= 0xA1 and byte <= 0xDF:
            ak.function1_call(&on_code_point, 0xFF61 - 0xA1 + (u32)byte)
            continue

        # 6. If byte is in the range 0x81 to 0x9F, inclusive, or 0xE0 to 0xFC, inclusive, set Shift_JIS lead to byte and return continue.
        if (byte >= 0x81 and byte <= 0x9F) or (byte >= 0xE0 and byte <= 0xFC):
            shift_jis_lead = byte
            continue

        # 7. Return error.
        ak.function1_call(&on_code_point, replacement_code_point)

# mark: EUC-KR =================================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:1559 TextCodec::EUCKRDecoder::process (override).
## https://encoding.spec.whatwg.org/#euc-kr-decoder
pub func euckr_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    # EUC-KR’s decoder has an associated EUC-KR lead (initially 0x00).
    var euc_kr_lead: u8 = 0x00

    # EUC-KR’s decoder’s handler, given ioQueue and byte, runs these steps:
    var index: usize = 0
    while true:
        # 1. If byte is end-of-queue and EUC-KR lead is not 0x00, set EUC-KR lead to 0x00 and return error.
        if index >= ak.string_view_length(&input) and euc_kr_lead != 0x00:
            euc_kr_lead = 0x00
            ak.function1_call(&on_code_point, replacement_code_point)
            continue

        # 2. If byte is end-of-queue and EUC-KR lead is 0x00, return finished.
        if index >= ak.string_view_length(&input) and euc_kr_lead == 0x00:
            return

        let byte = ak.string_view_at(&input, index)
        index += 1

        # 3. If EUC-KR lead is not 0x00, let lead be EUC-KR lead, let pointer be null, set EUC-KR lead to 0x00, and then:
        if euc_kr_lead != 0x00:
            let lead = euc_kr_lead
            var pointer: u32? = none
            euc_kr_lead = 0x00

            # 1. If byte is in the range 0x41 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − 0x41).
            if byte >= 0x41 and byte <= 0xFE:
                pointer = ((u32)lead - 0x81) * 190 + ((u32)byte - 0x41)

            # 2. Let code point be null if pointer is null, otherwise the index code point for pointer in index EUC-KR.
            var code_point: u32? = none
            if let p = pointer:
                code_point = index_euc_kr_code_point(p)

            # 3. If code point is non-null, return a code point whose value is code point.
            if let value = code_point:
                ak.function1_call(&on_code_point, value)
                continue

            # 4. If byte is an ASCII byte, restore byte to ioQueue.
            if byte <= 0x7F:
                index -= 1

            # 5. Return error.
            ak.function1_call(&on_code_point, replacement_code_point)
            continue

        # 4. If byte is an ASCII byte, return a code point whose value is byte.
        if byte <= 0x7F:
            ak.function1_call(&on_code_point, (u32)byte)
            continue

        # 5. If byte is in the range 0x81 to 0xFE, inclusive, set EUC-KR lead to byte and return continue.
        if byte >= 0x81 and byte <= 0xFE:
            euc_kr_lead = byte
            continue

        # 6. Return error.
        ak.function1_call(&on_code_point, replacement_code_point)

# mark: replacement ============================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:1626 TextCodec::ReplacementDecoder::process (override).
## https://encoding.spec.whatwg.org/#replacement-decoder
pub func replacement_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
    # replacement’s decoder has an associated replacement error returned (initially false).
    # replacement’s decoder’s handler, given ioQueue and byte, runs these steps:
    # 1. If byte is end-of-queue, return finished.
    # 2. If replacement error returned is false, set replacement error returned to true and return error.
    # 3. Return finished.

    if not ak.string_view_is_empty(&input):
        ak.function1_call(&on_code_point, replacement_code_point)

## Ported from Libraries/LibTextCodec/Decoder.h:134 TextCodec::ReplacementDecoder::validate (override).
pub func replacement_decoder_validate_impl(base: Decoder*, input: ak.StringView) -> bool:
    return ak.string_view_is_empty(&input)

# mark: Isomorphic decode ======================================================================

## Ported from Libraries/LibTextCodec/Decoder.cpp:1640 TextCodec::isomorphic_decode: the string whose
## code points are the input's bytes (decoding as ISO-8859-1 / Latin-1).
## https://infra.spec.whatwg.org/#isomorphic-decode
pub func isomorphic_decode(input: ak.StringView) -> ak.String:
    # To isomorphic decode a byte sequence input, return a string whose code point length is equal to input’s length
    # and whose code points have the same values as the values of input’s bytes, in the same order.
    # NB: This is essentially spec-speak for "Decode as ISO-8859-1 / Latin-1".
    var builder: ak.StringBuilder
    ak.string_builder_construct_usize(&builder, ak.string_view_length(&input))

    for byte in ak.string_view_bytes(&input):
        ak.string_builder_append_code_point(&builder, (u32)byte)

    return ak.string_builder_to_string_without_validation(&builder)