dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / src / text_codec / decoder_5.lucb
#==============================================================================================
#
# decoder_5 - the ISO-2022-JP, Shift_JIS, EUC-KR and replacement decoders, isomorphic decode
#
# DESCRIPTION:
# Port of Libraries/LibTextCodec/Decoder.cpp (lines 1224-1653) with ReplacementDecoder's
# inline validate from Decoder.h: the decoders of ISO-2022-JP (a state machine over escape
# sequences), Shift_JIS, EUC-KR and replacement, and Infra's isomorphic decode.
#
#==============================================================================================
# mark: ISO-2022-JP ============================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1235 TextCodec::ISO2022JPDecoder::process (override).
## https://encoding.spec.whatwg.org/#iso-2022-jp-decoder
pub func iso2022_jp_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# ISO-2022-JP’s decoder has an associated ISO-2022-JP decoder state (initially ASCII), ISO-2022-JP decoder output state (initially ASCII), ISO-2022-JP lead (initially 0x00), and ISO-2022-JP output (initially false).
var decoder_state = Iso2022JpState.ascii
var output_state = Iso2022JpState.ascii
var iso2022_jp_lead: u8 = 0x00
var iso2022_jp_output = false
var index: usize = 0
while true:
var byte: u8? = none
if index < ak.string_view_length(&input):
byte = ak.string_view_at(&input, index)
index += 1
let byte_value = byte else 0x00 # byte.value(), read only where byte has a value
# ISO-2022-JP’s decoder’s handler, given ioQueue and byte, runs these steps, switching on ISO-2022-JP decoder state:
match decoder_state:
.ascii:
# Based on byte:
# 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if byte == 0x1B:
decoder_state = Iso2022JpState.escape_start
continue
# 0x00 to 0x7F, excluding 0x0E, 0x0F, and 0x1B: Set ISO-2022-JP output to false and return a code point whose value is byte.
if byte != none and byte_value <= 0x7F and byte != 0x0E and byte != 0x0F:
iso2022_jp_output = false
ak.function1_call(&on_code_point, (u32)byte_value)
continue
# end-of-queue: Return finished.
if byte == none:
return
# Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false
ak.function1_call(&on_code_point, replacement_code_point)
.roman:
# Based on byte:
# 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if byte == 0x1B:
decoder_state = Iso2022JpState.escape_start
continue
# 0x5C: Set ISO-2022-JP output to false and return code point U+00A5.
if byte == 0x5C:
iso2022_jp_output = false
ak.function1_call(&on_code_point, 0x00A5)
continue
# 0x7E: Set ISO-2022-JP output to false and return code point U+203E.
if byte == 0x7E:
iso2022_jp_output = false
ak.function1_call(&on_code_point, 0x203E)
continue
# 0x00 to 0x7F, excluding 0x0E, 0x0F, 0x1B, 0x5C, and 0x7E: Set ISO-2022-JP output to false and return a code point whose value is byte.
if byte != none and byte_value <= 0x7F and byte != 0x0E and byte != 0x0F:
iso2022_jp_output = false
ak.function1_call(&on_code_point, (u32)byte_value)
continue
# end-of-queue: Return finished.
if byte == none:
return
# Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false
ak.function1_call(&on_code_point, replacement_code_point)
.katakana:
# Based on byte:
# 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if byte == 0x1B:
decoder_state = Iso2022JpState.escape_start
continue
# 0x21 to 0x5F: Set ISO-2022-JP output to false and return a code point whose value is 0xFF61 − 0x21 + byte.
if byte != none and byte_value >= 0x21 and byte_value <= 0x5F:
iso2022_jp_output = false
ak.function1_call(&on_code_point, 0xFF61 - 0x21 + (u32)byte_value)
continue
# end-of-queue: Return finished.
if byte == none:
return
# Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false
ak.function1_call(&on_code_point, replacement_code_point)
.lead_byte:
# Based on byte:
# 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if byte == 0x1B:
decoder_state = Iso2022JpState.escape_start
continue
# 0x21 to 0x7E: Set ISO-2022-JP output to false, ISO-2022-JP lead to byte, ISO-2022-JP decoder state to trail byte, and return continue.
if byte != none and byte_value >= 0x21 and byte_value <= 0x7E:
iso2022_jp_output = false
iso2022_jp_lead = byte_value
decoder_state = Iso2022JpState.trail_byte
continue
# end-of-queue: Return finished.
if byte == none:
return
# Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false
ak.function1_call(&on_code_point, replacement_code_point)
.trail_byte:
# Based on byte:
# 0x1B: Set ISO-2022-JP decoder state to escape start and return error.
if byte == 0x1B:
decoder_state = Iso2022JpState.escape_start
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 0x21 to 0x7E:
if byte != none and byte_value >= 0x21 and byte_value <= 0x7E:
# 1. Set the ISO-2022-JP decoder state to lead byte.
decoder_state = Iso2022JpState.lead_byte
# 2. Let pointer be (ISO-2022-JP lead − 0x21) × 94 + byte − 0x21.
let pointer: u32 = ((u32)iso2022_jp_lead - 0x21) * 94 + (u32)byte_value - 0x21
# 3. Let code point be the index code point for pointer in index jis0208.
let code_point = index_jis0208_code_point(pointer)
# 4. If code point is null, return error.
if code_point == none:
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 5. Return a code point whose value is code point.
ak.function1_call(&on_code_point, code_point else trap("VERIFY: Optional::value() of an empty Optional"))
continue
# end-of-queue: Set the ISO-2022-JP decoder state to lead byte and return error.
if byte == none:
decoder_state = Iso2022JpState.lead_byte
ak.function1_call(&on_code_point, replacement_code_point)
continue
# Otherwise: Set ISO-2022-JP decoder state to lead byte and return error.
decoder_state = Iso2022JpState.lead_byte
ak.function1_call(&on_code_point, replacement_code_point)
.escape_start:
# 1. If byte is either 0x24 or 0x28, set ISO-2022-JP lead to byte, ISO-2022-JP decoder state to escape, and return continue.
if byte == 0x24 or byte == 0x28:
iso2022_jp_lead = byte_value
decoder_state = Iso2022JpState.escape
continue
# 2. If byte is not end-of-queue, then restore byte to ioQueue.
if byte != none:
index -= 1
# 3. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state, and return error.
iso2022_jp_output = false
decoder_state = output_state
ak.function1_call(&on_code_point, replacement_code_point)
.escape:
# 1. Let lead be ISO-2022-JP lead and set ISO-2022-JP lead to 0x00.
let lead = iso2022_jp_lead
iso2022_jp_lead = 0x00
# 2. Let state be null.
var state: Iso2022JpState? = none
# 3. If lead is 0x28 and byte is 0x42, set state to ASCII.
if lead == 0x28 and byte == 0x42:
state = Iso2022JpState.ascii
# 4. If lead is 0x28 and byte is 0x4A, set state to Roman.
if lead == 0x28 and byte == 0x4A:
state = Iso2022JpState.roman
# 5. If lead is 0x28 and byte is 0x49, set state to katakana.
if lead == 0x28 and byte == 0x49:
state = Iso2022JpState.katakana
# 6. If lead is 0x24 and byte is either 0x40 or 0x42, set state to lead byte.
if lead == 0x24 and (byte == 0x40 or byte == 0x42):
state = Iso2022JpState.lead_byte
# 7. If state is non-null, then:
if let new_state = state:
# 1. Set ISO-2022-JP decoder state and ISO-2022-JP decoder output state to state.
decoder_state = new_state
output_state = new_state
# 2. Let output be the value of ISO-2022-JP output.
let output = iso2022_jp_output
# 3. Set ISO-2022-JP output to true.
iso2022_jp_output = true
# 4. Return continue, if output is false, and error otherwise.
if output:
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 8. If byte is end-of-queue, then restore lead to ioQueue; otherwise, restore « lead, byte » to ioQueue.
index -= 2 if byte != none else 1
# 9. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state and return error.
iso2022_jp_output = false
decoder_state = output_state
ak.function1_call(&on_code_point, replacement_code_point)
_:
trap("VERIFY_NOT_REACHED: ISO2022JPState")
# mark: Shift_JIS ==============================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1474 TextCodec::ShiftJISDecoder::process (override).
## https://encoding.spec.whatwg.org/#shift_jis-decoder
pub func shift_jis_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# Shift_JIS’s decoder has an associated Shift_JIS lead (initially 0x00).
var shift_jis_lead: u8 = 0x00
# Shift_JIS’s decoder’s handler, given ioQueue and byte, runs these steps:
var index: usize = 0
while true:
# 1. If byte is end-of-queue and Shift_JIS lead is not 0x00, set Shift_JIS lead to 0x00 and return error.
if index >= ak.string_view_length(&input) and shift_jis_lead != 0x00:
shift_jis_lead = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 2. If byte is end-of-queue and Shift_JIS lead is 0x00, return finished.
if index >= ak.string_view_length(&input) and shift_jis_lead == 0x00:
return
let byte = ak.string_view_at(&input, index)
index += 1
# 3. If Shift_JIS lead is not 0x00, let lead be Shift_JIS lead, let pointer be null, set Shift_JIS lead to 0x00, and then:
if shift_jis_lead != 0x00:
let lead = shift_jis_lead
var pointer: u32? = none
shift_jis_lead = 0x00
# 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
let offset: u8 = 0x40 if byte < 0x7F else 0x41
# 2. Let lead offset be 0x81 if lead is less than 0xA0, otherwise 0xC1.
let lead_offset: u8 = 0x81 if lead < 0xA0 else 0xC1
# 3. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFC, inclusive, set pointer to (lead − lead offset) × 188 + byte − offset.
if (byte >= 0x40 and byte <= 0x7E) or (byte >= 0x80 and byte <= 0xFC):
pointer = (u32)(lead - lead_offset) * 188 + (u32)byte - (u32)offset
# 4. If pointer is in the range 8836 to 10715, inclusive, return a code point whose value is 0xE000 − 8836 + pointer.
if let p = pointer:
if p >= 8836 and p <= 10715:
ak.function1_call(&on_code_point, 0xE000 - 8836 + p)
continue
# 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index jis0208.
var code_point: u32? = none
if let p = pointer:
code_point = index_jis0208_code_point(p)
# 6. If code point is non-null, return a code point whose value is code point.
if let value = code_point:
ak.function1_call(&on_code_point, value)
continue
# 7. If byte is an ASCII byte, restore byte to ioQueue.
if byte <= 0x7F:
index -= 1
# 8. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 4. If byte is an ASCII byte or 0x80, return a code point whose value is byte.
if byte <= 0x80:
ak.function1_call(&on_code_point, (u32)byte)
continue
# 5. If byte is in the range 0xA1 to 0xDF, inclusive, return a code point whose value is 0xFF61 − 0xA1 + byte.
if byte >= 0xA1 and byte <= 0xDF:
ak.function1_call(&on_code_point, 0xFF61 - 0xA1 + (u32)byte)
continue
# 6. If byte is in the range 0x81 to 0x9F, inclusive, or 0xE0 to 0xFC, inclusive, set Shift_JIS lead to byte and return continue.
if (byte >= 0x81 and byte <= 0x9F) or (byte >= 0xE0 and byte <= 0xFC):
shift_jis_lead = byte
continue
# 7. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
# mark: EUC-KR =================================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1559 TextCodec::EUCKRDecoder::process (override).
## https://encoding.spec.whatwg.org/#euc-kr-decoder
pub func euckr_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# EUC-KR’s decoder has an associated EUC-KR lead (initially 0x00).
var euc_kr_lead: u8 = 0x00
# EUC-KR’s decoder’s handler, given ioQueue and byte, runs these steps:
var index: usize = 0
while true:
# 1. If byte is end-of-queue and EUC-KR lead is not 0x00, set EUC-KR lead to 0x00 and return error.
if index >= ak.string_view_length(&input) and euc_kr_lead != 0x00:
euc_kr_lead = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 2. If byte is end-of-queue and EUC-KR lead is 0x00, return finished.
if index >= ak.string_view_length(&input) and euc_kr_lead == 0x00:
return
let byte = ak.string_view_at(&input, index)
index += 1
# 3. If EUC-KR lead is not 0x00, let lead be EUC-KR lead, let pointer be null, set EUC-KR lead to 0x00, and then:
if euc_kr_lead != 0x00:
let lead = euc_kr_lead
var pointer: u32? = none
euc_kr_lead = 0x00
# 1. If byte is in the range 0x41 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − 0x41).
if byte >= 0x41 and byte <= 0xFE:
pointer = ((u32)lead - 0x81) * 190 + ((u32)byte - 0x41)
# 2. Let code point be null if pointer is null, otherwise the index code point for pointer in index EUC-KR.
var code_point: u32? = none
if let p = pointer:
code_point = index_euc_kr_code_point(p)
# 3. If code point is non-null, return a code point whose value is code point.
if let value = code_point:
ak.function1_call(&on_code_point, value)
continue
# 4. If byte is an ASCII byte, restore byte to ioQueue.
if byte <= 0x7F:
index -= 1
# 5. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 4. If byte is an ASCII byte, return a code point whose value is byte.
if byte <= 0x7F:
ak.function1_call(&on_code_point, (u32)byte)
continue
# 5. If byte is in the range 0x81 to 0xFE, inclusive, set EUC-KR lead to byte and return continue.
if byte >= 0x81 and byte <= 0xFE:
euc_kr_lead = byte
continue
# 6. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
# mark: replacement ============================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1626 TextCodec::ReplacementDecoder::process (override).
## https://encoding.spec.whatwg.org/#replacement-decoder
pub func replacement_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# replacement’s decoder has an associated replacement error returned (initially false).
# replacement’s decoder’s handler, given ioQueue and byte, runs these steps:
# 1. If byte is end-of-queue, return finished.
# 2. If replacement error returned is false, set replacement error returned to true and return error.
# 3. Return finished.
if not ak.string_view_is_empty(&input):
ak.function1_call(&on_code_point, replacement_code_point)
## Ported from Libraries/LibTextCodec/Decoder.h:134 TextCodec::ReplacementDecoder::validate (override).
pub func replacement_decoder_validate_impl(base: Decoder*, input: ak.StringView) -> bool:
return ak.string_view_is_empty(&input)
# mark: Isomorphic decode ======================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1640 TextCodec::isomorphic_decode: the string whose
## code points are the input's bytes (decoding as ISO-8859-1 / Latin-1).
## https://infra.spec.whatwg.org/#isomorphic-decode
pub func isomorphic_decode(input: ak.StringView) -> ak.String:
# To isomorphic decode a byte sequence input, return a string whose code point length is equal to input’s length
# and whose code points have the same values as the values of input’s bytes, in the same order.
# NB: This is essentially spec-speak for "Decode as ISO-8859-1 / Latin-1".
var builder: ak.StringBuilder
ak.string_builder_construct_usize(&builder, ak.string_view_length(&input))
for byte in ak.string_view_bytes(&input):
ak.string_builder_append_code_point(&builder, (u32)byte)
return ak.string_builder_to_string_without_validation(&builder)