dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / text_codec / decoder_4.lucb
#==============================================================================================
#
# decoder_4 - the gb18030, Big5 and EUC-JP decoders
#
# DESCRIPTION:
# Port of Libraries/LibTextCodec/Decoder.cpp (lines 843-1222): the index gb18030 ranges
# code point, and the decoders of gb18030 (and GBK), Big5 and EUC-JP, each a loop over the
# input running the Encoding Standard's handler with its state in locals; "restore byte to
# ioQueue" steps the index back.
#
#==============================================================================================
# mark: gb18030 ================================================================================
# index_gb18030_ranges_code_point's binary_search comparator:
# [](auto const pointer, auto const& entry) { return pointer - entry.pointer; } in u32, as int.
func gb18030_ranges_compare_pointer(context: void*?, pointer: const u32*, entry: const Gb18030RangeEntry*) -> i32:
return (i32)(*pointer -% entry.pointer)
# Ported from Libraries/LibTextCodec/Decoder.cpp:843 TextCodec::index_gb18030_ranges_code_point.
# https://encoding.spec.whatwg.org/#index-gb18030-ranges-code-point
func index_gb18030_ranges_code_point(pointer: u32) -> u32?:
# 1. If pointer is greater than 39419 and less than 189000, or pointer is greater than 1237575, return null.
if (pointer > 39419 and pointer < 189000) or pointer > 1237575:
return none
# 2. If pointer is 7457, return code point U+E7C7.
if pointer == 7457:
return 0xE7C7
# FIXME: Encoding specification is not updated to GB-18030-2022 yet (https://github.com/whatwg/encoding/issues/312)
# NOTE: This matches https://commits.webkit.org/266173@main
let gb18030_2022_code_point: u32? = match pointer:
19057 => 0xE81E # 82 35 90 37
19058 => 0xE826 # 82 35 90 38
19059 => 0xE82B # 82 35 90 39
19060 => 0xE82C # 82 35 91 30
19061 => 0xE832 # 82 35 91 31
19062 => 0xE843 # 82 35 91 32
19063 => 0xE854 # 82 35 91 33
19064 => 0xE864 # 82 35 91 34
39076 => 0xE78D # 84 31 82 36
39077 => 0xE78F # 84 31 82 37
39078 => 0xE78E # 84 31 82 38
39079 => 0xE790 # 84 31 82 39
39080 => 0xE791 # 84 31 83 30
39081 => 0xE792 # 84 31 83 31
39082 => 0xE793 # 84 31 83 32
39083 => 0xE794 # 84 31 83 33
39084 => 0xE795 # 84 31 83 34
39085 => 0xE796 # 84 31 83 35
_ => none # default: break;
if let code_point = gb18030_2022_code_point:
return code_point
# 3. Let offset be the last pointer in index gb18030 ranges that is less than or equal to pointer and let code point offset be its corresponding code point.
var last_index: usize = 0
let comparator = ak.Function2[const u32*, const Gb18030RangeEntry*, i32](call = gb18030_ranges_compare_pointer, context = none)
_ = ak.binary_search(s_gb18030_ranges, pointer, &last_index, comparator)
let offset = s_gb18030_ranges[last_index].pointer
let code_point_offset = s_gb18030_ranges[last_index].code_point
# 4. Return a code point whose value is code point offset + pointer − offset.
return code_point_offset + pointer - offset
## Ported from Libraries/LibTextCodec/Decoder.cpp:909 TextCodec::GB18030Decoder::process (override).
## https://encoding.spec.whatwg.org/#gb18030-decoder
pub func gb18030_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# gb18030’s decoder has an associated gb18030 first, gb18030 second, and gb18030 third (all initially 0x00).
var first: u8 = 0x00
var second: u8 = 0x00
var third: u8 = 0x00
# gb18030’s decoder’s handler, given ioQueue and byte, runs these steps:
var index: usize = 0
while true:
# 1. If byte is end-of-queue and gb18030 first, gb18030 second, and gb18030 third are 0x00, return finished.
if index >= ak.string_view_length(&input) and first == 0x00 and second == 0x00 and third == 0x00:
return
# 2. If byte is end-of-queue, and gb18030 first, gb18030 second, or gb18030 third is not 0x00, set gb18030 first, gb18030 second, and gb18030 third to 0x00, and return error.
if index >= ak.string_view_length(&input) and (first != 0x00 or second != 0x00 or third != 0x00):
first = 0x00
second = 0x00
third = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
let byte = ak.string_view_at(&input, index)
index += 1
# 3. If gb18030 third is not 0x00, then:
if third != 0x00:
# 1. If byte is not in the range 0x30 to 0x39, inclusive, then:
if byte < 0x30 or byte > 0x39:
# 1. Restore « gb18030 second, gb18030 third, byte » to ioQueue.
index -= 3
# 2. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
first = 0x00
second = 0x00
third = 0x00
# 3. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 2. Let code point be the index gb18030 ranges code point for ((gb18030 first − 0x81) × (10 × 126 × 10)) + ((gb18030 second − 0x30) × (10 × 126)) + ((gb18030 third − 0x81) × 10) + byte − 0x30.
let code_point = index_gb18030_ranges_code_point((((u32)first - 0x81) * (10 * 126 * 10)) + (((u32)second - 0x30) * (10 * 126)) + (((u32)third - 0x81) * 10) + (u32)byte - 0x30)
# 3. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
first = 0x00
second = 0x00
third = 0x00
# 4. If code point is null, return error.
if code_point == none:
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 5. Return a code point whose value is code point.
ak.function1_call(&on_code_point, code_point else trap("VERIFY: Optional::value() of an empty Optional"))
continue
# 4. If gb18030 second is not 0x00, then:
if second != 0x00:
# 1. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 third to byte and return continue.
if byte >= 0x81 and byte <= 0xFE:
third = byte
continue
# 2. Restore « gb18030 second, byte » to ioQueue, set gb18030 first and gb18030 second to 0x00, and return error.
index -= 2
first = 0x00
second = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 5. If gb18030 first is not 0x00, then:
if first != 0x00:
# 1. If byte is in the range 0x30 to 0x39, inclusive, set gb18030 second to byte and return continue.
if byte >= 0x30 and byte <= 0x39:
second = byte
continue
# 2. Let lead be gb18030 first, let pointer be null, and set gb18030 first to 0x00.
let lead = first
var pointer: u32? = none
first = 0x00
# 3. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
let offset: u8 = 0x40 if byte < 0x7F else 0x41
# 4. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − offset).
if (byte >= 0x40 and byte <= 0x7E) or (byte >= 0x80 and byte <= 0xFE):
pointer = ((u32)lead - 0x81) * 190 + (u32)(byte - offset)
# 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index gb18030.
var code_point: u32? = none
if let p = pointer:
code_point = index_gb18030_code_point(p)
# 6. If code point is non-null, return a code point whose value is code point.
if let value = code_point:
ak.function1_call(&on_code_point, value)
continue
# 7. If byte is an ASCII byte, restore byte to ioQueue.
if byte <= 0x7F:
index -= 1
# 8. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 6. If byte is an ASCII byte, return a code point whose value is byte.
if byte <= 0x7F:
ak.function1_call(&on_code_point, (u32)byte)
continue
# 7. If byte is 0x80, return code point U+20AC.
if byte == 0x80:
ak.function1_call(&on_code_point, 0x20AC)
continue
# 8. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 first to byte and return continue.
if byte >= 0x81 and byte <= 0xFE:
first = byte
continue
# 9. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
# mark: Big5 ===================================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1047 TextCodec::Big5Decoder::process (override).
## https://encoding.spec.whatwg.org/#big5-decoder
pub func big5_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# Big5’s decoder has an associated Big5 lead (initially 0x00).
var big5_lead: u8 = 0x00
# Big5’s decoder’s handler, given ioQueue and byte, runs these steps:
var index: usize = 0
while true:
# 1. If byte is end-of-queue and Big5 lead is not 0x00, set Big5 lead to 0x00 and return error.
if index >= ak.string_view_length(&input) and big5_lead != 0x00:
big5_lead = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 2. If byte is end-of-queue and Big5 lead is 0x00, return finished.
if index >= ak.string_view_length(&input) and big5_lead == 0x00:
return
let byte = ak.string_view_at(&input, index)
index += 1
# 3. If Big5 lead is not 0x00, let lead be Big5 lead, let pointer be null, set Big5 lead to 0x00, and then:
if big5_lead != 0x00:
let lead = big5_lead
var pointer: u32? = none
big5_lead = 0x00
# 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x62.
let offset: u8 = 0x40 if byte < 0x7F else 0x62
# 2. If byte is in the range 0x40 to 0x7E, inclusive, or 0xA1 to 0xFE, inclusive, set pointer to (lead − 0x81) × 157 + (byte − offset).
if (byte >= 0x40 and byte <= 0x7E) or (byte >= 0xA1 and byte <= 0xFE):
pointer = ((u32)lead - 0x81) * 157 + (u32)(byte - offset)
# 3. If there is a row in the table below whose first column is pointer, return the two code points listed in its second column (the third column is irrelevant):
if pointer == 1133:
ak.function1_call(&on_code_point, 0x00CA)
ak.function1_call(&on_code_point, 0x0304)
continue
if pointer == 1135:
ak.function1_call(&on_code_point, 0x00CA)
ak.function1_call(&on_code_point, 0x030C)
continue
if pointer == 1164:
ak.function1_call(&on_code_point, 0x00EA)
ak.function1_call(&on_code_point, 0x0304)
continue
if pointer == 1166:
ak.function1_call(&on_code_point, 0x00EA)
ak.function1_call(&on_code_point, 0x030C)
continue
# 4. Let code point be null if pointer is null, otherwise the index code point for pointer in index Big5.
var code_pointer: u32? = none
if let p = pointer:
code_pointer = index_big5_code_point(p)
# 5. If code point is non-null, return a code point whose value is code point.
if let value = code_pointer:
ak.function1_call(&on_code_point, value)
continue
# 6. If byte is an ASCII byte, restore byte to ioQueue.
if byte <= 0x7F:
index -= 1
# 7. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 4. If byte is an ASCII byte, return a code point whose value is byte.
if byte <= 0x7F:
ak.function1_call(&on_code_point, (u32)byte)
continue
# 5. If byte is in the range 0x81 to 0xFE, inclusive, set Big5 lead to byte and return continue.
if byte >= 0x81 and byte <= 0xFE:
big5_lead = byte
continue
# 6. Return error
ak.function1_call(&on_code_point, replacement_code_point)
# mark: EUC-JP =================================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:1139 TextCodec::EUCJPDecoder::process (override).
## https://encoding.spec.whatwg.org/#euc-jp-decoder
pub func eucjp_decoder_process_impl(base: Decoder*, input: ak.StringView, on_code_point: ak.Function1[u32, unit]) -> !:
# EUC-JP’s decoder has an associated EUC-JP jis0212 (initially false) and EUC-JP lead (initially 0x00).
var jis0212 = false
var euc_jp_lead: u8 = 0x00
# EUC-JP’s decoder’s handler, given ioQueue and byte, runs these steps:
var index: usize = 0
while true:
# 1. If byte is end-of-queue and EUC-JP lead is not 0x00, set EUC-JP lead to 0x00, and return error.
if index >= ak.string_view_length(&input) and euc_jp_lead != 0x00:
euc_jp_lead = 0x00
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 2. If byte is end-of-queue and EUC-JP lead is 0x00, return finished.
if index >= ak.string_view_length(&input) and euc_jp_lead == 0x00:
return
let byte = ak.string_view_at(&input, index)
index += 1
# 3. If EUC-JP lead is 0x8E and byte is in the range 0xA1 to 0xDF, inclusive, set EUC-JP lead to 0x00 and return a code point whose value is 0xFF61 − 0xA1 + byte.
if euc_jp_lead == 0x8E and byte >= 0xA1 and byte <= 0xDF:
euc_jp_lead = 0x00
ak.function1_call(&on_code_point, 0xFF61 - 0xA1 + (u32)byte)
continue
# 4. If EUC-JP lead is 0x8F and byte is in the range 0xA1 to 0xFE, inclusive, set EUC-JP jis0212 to true, set EUC-JP lead to byte, and return continue.
if euc_jp_lead == 0x8F and byte >= 0xA1 and byte <= 0xFE:
jis0212 = true
euc_jp_lead = byte
continue
# 5. If EUC-JP lead is not 0x00, let lead be EUC-JP lead, set EUC-JP lead to 0x00, and then:
if euc_jp_lead != 0x00:
let lead = euc_jp_lead
euc_jp_lead = 0x00
# 1. Let code point be null.
var code_point: u32? = none
# 2. If lead and byte are both in the range 0xA1 to 0xFE, inclusive, then set code point to the index code point for (lead − 0xA1) × 94 + byte − 0xA1 in index jis0208 if EUC-JP jis0212 is false and in index jis0212 otherwise.
if lead >= 0xA1 and lead <= 0xFE and byte >= 0xA1 and byte <= 0xFE:
let pointer = ((u32)lead - 0xA1) * 94 + (u32)byte - 0xA1
code_point = index_jis0212_code_point(pointer) if jis0212 else index_jis0208_code_point(pointer)
# 3. Set EUC-JP jis0212 to false.
jis0212 = false
# 4. If code point is non-null, return a code point whose value is code point.
if let value = code_point:
ak.function1_call(&on_code_point, value)
continue
# 5. If byte is an ASCII byte, restore byte to ioQueue.
if byte <= 0x7F:
index -= 1
# 6. Return error.
ak.function1_call(&on_code_point, replacement_code_point)
continue
# 6. If byte is an ASCII byte, return a code point whose value is byte.
if byte <= 0x7F:
ak.function1_call(&on_code_point, (u32)byte)
continue
# 7. If byte is 0x8E, 0x8F, or in the range 0xA1 to 0xFE, inclusive, set EUC-JP lead to byte and return continue.
if byte == 0x8E or byte == 0x8F or (byte >= 0xA1 and byte <= 0xFE):
euc_jp_lead = byte
continue
# 8. Return error.
ak.function1_call(&on_code_point, replacement_code_point)