dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / src / text_codec / encoder_2.lucb
#==============================================================================================
#
# encoder_2 - the Shift_JIS, EUC-KR, Big5, gb18030 and single-byte encoders, isomorphic encode
#
# DESCRIPTION:
# Port of Libraries/LibTextCodec/Encoder.cpp (lines 338-689) with the inline constructor of
# Encoder.h's SingleByteEncoder: the index pointer lookups that skip or search parts of an
# index, the encoders of Shift_JIS, EUC-KR, Big5, gb18030 (and GBK) and the single-byte
# encodings, and Infra's isomorphic encode.
#
#==============================================================================================
# mark: Shift_JIS ==============================================================================
# Ported from Libraries/LibTextCodec/Encoder.cpp:338 TextCodec::code_point_jis0208_index_skipping_range.
func code_point_jis0208_index_skipping_range(code_point: u32, skip_from: u32, skip_to: u32) -> u32?:
assert(skip_to >= skip_from)
for i in 0..<s_jis0208_index.length:
if i >= skip_from and i <= skip_to:
continue
if s_jis0208_index[i] == code_point:
return (u32)i
return none
# Ported from Libraries/LibTextCodec/Encoder.cpp:351 TextCodec::index_shift_jis_pointer.
# https://encoding.spec.whatwg.org/#index-shift_jis-pointer
func index_shift_jis_pointer(code_point: u32) -> u32?:
# 1. Let index be index jis0208 excluding all entries whose pointer is in the range 8272 to 8835, inclusive.
let pointer = code_point_jis0208_index_skipping_range(code_point, 8272, 8835)
if pointer == none:
return none
# 2. Return the index pointer for code point in index.
return pointer
## Ported from Libraries/LibTextCodec/Encoder.cpp:363 TextCodec::ShiftJISEncoder::process (override).
## https://encoding.spec.whatwg.org/#shift_jis-encoder
pub func shift_jis_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
# for (u32 item : input)
let end = ak.utf8_view_end(&input)
var iterator = ak.utf8_view_begin(&input)
while iterator != end:
var item = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
# 1. If code point is end-of-queue, return finished.
# 2. If code point is an ASCII code point or U+0080, return a byte whose value is code point.
if ak.is_ascii(item) or item == 0x0080:
ak.function1_call(&on_byte, (u8)item)
continue
# 3. If code point is U+00A5, return byte 0x5C.
if item == 0x00A5:
ak.function1_call(&on_byte, (u8)0x5C)
continue
# 4. If code point is U+203E, return byte 0x7E.
if item == 0x203E:
ak.function1_call(&on_byte, (u8)0x7E)
continue
# 5. If code point is in the range U+FF61 to U+FF9F, inclusive, return a byte whose value is code point − 0xFF61 + 0xA1.
if item >= 0xFF61 and item <= 0xFF9F:
ak.function1_call(&on_byte, (u8)(item - 0xFF61 + 0xA1))
continue
# 6. If code point is U+2212, set it to U+FF0D.
if item == 0x2212:
item = 0xFF0D
# 7. Let pointer be the index Shift_JIS pointer for code point.
let pointer = index_shift_jis_pointer(item)
# 8. If pointer is null, return error with code point.
if pointer == none:
ak.function1_call(&on_error, item)
continue
let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")
# 9. Let lead be pointer / 188.
let lead = index_pointer // 188
# 10. Let lead offset be 0x81 if lead is less than 0x1F, otherwise 0xC1.
var lead_offset: u32 = 0xC1
if lead < 0x1F:
lead_offset = 0x81
# 11. Let trail be pointer % 188.
let trail = index_pointer % 188
# 12. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
var offset: u32 = 0x41
if trail < 0x3F:
offset = 0x40
# 13. Return two bytes whose values are lead + lead offset and trail + offset.
ak.function1_call(&on_byte, (u8)(lead + lead_offset))
ak.function1_call(&on_byte, (u8)(trail + offset))
# mark: EUC-KR =================================================================================
## Ported from Libraries/LibTextCodec/Encoder.cpp:430 TextCodec::EUCKREncoder::process (override).
## https://encoding.spec.whatwg.org/#euc-kr-encoder
pub func euckr_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
# for (u32 item : input)
let end = ak.utf8_view_end(&input)
var iterator = ak.utf8_view_begin(&input)
while iterator != end:
let item = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
# 1. If code point is end-of-queue, return finished.
# 2. If code point is an ASCII code point, return a byte whose value is code point.
if ak.is_ascii(item):
ak.function1_call(&on_byte, (u8)item)
continue
# 3. Let pointer be the index pointer for code point in index EUC-KR.
let pointer = code_point_euc_kr_index(item)
# 4. If pointer is null, return error with code point.
if pointer == none:
ak.function1_call(&on_error, item)
continue
let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")
# 5. Let lead be pointer / 190 + 0x81.
let lead = index_pointer // 190 + 0x81
# 6. Let trail be pointer % 190 + 0x41.
let trail = index_pointer % 190 + 0x41
# 7. Return two bytes whose values are lead and trail.
ak.function1_call(&on_byte, (u8)lead)
ak.function1_call(&on_byte, (u8)trail)
# mark: Big5 ===================================================================================
# Ported from Libraries/LibTextCodec/Encoder.cpp:465 TextCodec::index_big5_pointer.
# https://encoding.spec.whatwg.org/#index-big5-pointer
func index_big5_pointer(code_point: u32) -> u32?:
# 1. Let index be index Big5 excluding all entries whose pointer is less than (0xA1 - 0x81) × 157.
let start_index: u32 = (0xA1 - 0x81) * 157 - s_big5_index_first_pointer
# 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345, return the last pointer
# corresponding to code point in index.
let last_pointer_code_points: u32[6] = [0x2550, 0x255E, 0x2561, 0x256A, 0x5341, 0x5345]
var contains = false # Array<u32, 6> { ... }.contains_slow(code_point)
for candidate in last_pointer_code_points:
if candidate == code_point:
contains = true
if contains:
var i: u32 = (u32)s_big5_index.length - 1
while i >= start_index:
if s_big5_index[i] == code_point:
return s_big5_index_first_pointer + i
i -= 1
return none
# 3. Return the index pointer for code point in index.
var i: u32 = start_index
while i < s_big5_index.length:
if s_big5_index[i] == code_point:
return s_big5_index_first_pointer + i
i += 1
return none
## Ported from Libraries/LibTextCodec/Encoder.cpp:491 TextCodec::Big5Encoder::process (override).
## https://encoding.spec.whatwg.org/#big5-encoder
pub func big5_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
# for (u32 item : input)
let end = ak.utf8_view_end(&input)
var iterator = ak.utf8_view_begin(&input)
while iterator != end:
let item = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
# 1. If code point is end-of-queue, return finished.
# 2. If code point is an ASCII code point, return a byte whose value is code point.
if ak.is_ascii(item):
ak.function1_call(&on_byte, (u8)item)
continue
# 3. Let pointer be the index Big5 pointer for code point.
let pointer = index_big5_pointer(item)
# 4. If pointer is null, return error with code point.
if pointer == none:
ak.function1_call(&on_error, item)
continue
let index_pointer = pointer else trap("VERIFY: Optional::operator* of an empty Optional")
# 5. Let lead be pointer / 157 + 0x81.
let lead = index_pointer // 157 + 0x81
# 6. Let trail be pointer % 157.
let trail = index_pointer % 157
# 7. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x62.
var offset: u32 = 0x62
if trail < 0x3f:
offset = 0x40
# 8. Return two bytes whose values are lead and trail + offset.
ak.function1_call(&on_byte, (u8)lead)
ak.function1_call(&on_byte, (u8)(trail + offset))
# mark: gb18030 ================================================================================
# index_gb18030_ranges_pointer's binary_search comparator:
# [](auto const code_point, auto const& entry) { return code_point - entry.code_point; } in u32, as int.
func gb18030_ranges_compare_code_point(context: void*?, code_point: const u32*, entry: const Gb18030RangeEntry*) -> i32:
return (i32)(*code_point -% entry.code_point)
# Ported from Libraries/LibTextCodec/Encoder.cpp:531 TextCodec::index_gb18030_ranges_pointer.
# https://encoding.spec.whatwg.org/#index-gb18030-ranges-pointer
func index_gb18030_ranges_pointer(code_point: u32) -> u32:
# 1. If code point is U+E7C7, return pointer 7457.
if code_point == 0xe7c7:
return 7457
# 2. Let offset be the last code point in index gb18030 ranges that is less than
# or equal to code point and let pointer offset be its corresponding pointer.
var last_index: usize = 0
let comparator = ak.Function2[const u32*, const Gb18030RangeEntry*, i32](call = gb18030_ranges_compare_code_point, context = none)
_ = ak.binary_search(s_gb18030_ranges, code_point, &last_index, comparator)
let offset = s_gb18030_ranges[last_index].code_point
let pointer_offset = s_gb18030_ranges[last_index].pointer
# 3. Return a pointer whose value is pointer offset + code point − offset.
return pointer_offset + code_point - offset
## Ported from Libraries/LibTextCodec/Encoder.cpp:550 TextCodec::GB18030Encoder::GB18030Encoder: a
## gb18030 encoder, or the GBK encoder when is_gbk is yes.
pub func gb18030_encoder_construct(this: Gb18030Encoder*, is_gbk: Gb18030EncoderIsGbk = Gb18030EncoderIsGbk.no):
gb18030_encoder_init_fields(this)
this.encoder_base.vtable = &gb18030_encoder_vtable.encoder_vtable # the C++ vptr
this.encoder_base.class_id = gb18030_encoder_class.first_id
this.m_is_gbk = is_gbk
## Ported from Libraries/LibTextCodec/Encoder.cpp:556 TextCodec::GB18030Encoder::process (override).
## https://encoding.spec.whatwg.org/#gb18030-encoder
pub func gb18030_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
let this = (Gb18030Encoder*)base
let gbk = this.m_is_gbk == Gb18030EncoderIsGbk.yes
# for (u32 item : input)
let end = ak.utf8_view_end(&input)
var iterator = ak.utf8_view_begin(&input)
while iterator != end:
let item = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
# 1. If code point is end-of-queue, return finished.
# 2. If code point is an ASCII code point, return a byte whose value is code point.
if ak.is_ascii(item):
ak.function1_call(&on_byte, (u8)item)
continue
# 3. If code point is U+E5E5, return error with code point.
if item == 0xE5E5:
ak.function1_call(&on_error, item)
continue
# 4. If is GBK is true and code point is U+20AC, return byte 0x80.
if gbk and item == 0x20AC:
ak.function1_call(&on_byte, (u8)0x80)
continue
# 5. Let pointer be the index pointer for code point in index gb18030.
let pointer = code_point_gb18030_index(item)
# 6. If pointer is non-null, then:
if let index_pointer = pointer:
# 1. Let lead be pointer / 190 + 0x81.
let lead = index_pointer // 190 + 0x81
# 2. Let trail be pointer % 190.
let trail = index_pointer % 190
# 3. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
var offset: u32 = 0x41
if trail < 0x3f:
offset = 0x40
# 4. Return two bytes whose values are lead and trail + offset.
ak.function1_call(&on_byte, (u8)lead)
ak.function1_call(&on_byte, (u8)(trail + offset))
continue
# 7. If is GBK is true, return error with code point.
if gbk:
ak.function1_call(&on_error, item)
continue
# 8. Set pointer to the index gb18030 ranges pointer for code point.
var ranges_pointer = index_gb18030_ranges_pointer(item)
# 9. Let byte1 be pointer / (10 × 126 × 10).
let byte1 = ranges_pointer // (10 * 126 * 10)
# 10. Set pointer to pointer % (10 × 126 × 10).
ranges_pointer = ranges_pointer % (10 * 126 * 10)
# 11. Let byte2 be pointer / (10 × 126).
let byte2 = ranges_pointer // (10 * 126)
# 12. Set pointer to pointer % (10 × 126).
ranges_pointer = ranges_pointer % (10 * 126)
# 13. Let byte3 be pointer / 10.
let byte3 = ranges_pointer // 10
# 14. Let byte4 be pointer % 10.
let byte4 = ranges_pointer % 10
# 15. Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30.
ak.function1_call(&on_byte, (u8)(byte1 + 0x81))
ak.function1_call(&on_byte, (u8)(byte2 + 0x30))
ak.function1_call(&on_byte, (u8)(byte3 + 0x81))
ak.function1_call(&on_byte, (u8)(byte4 + 0x30))
# mark: Single-byte encoder ====================================================================
# The donor instantiates SingleByteEncoder<ArrayType> only with ArrayType = u16 (its tables are
# Array<u16, 128>), and a Base generic body cannot compare a T with a u32, so the port's functions
# take the u16 instantiation.
## Ported from Libraries/LibTextCodec/Encoder.h:80 TextCodec::SingleByteEncoder::SingleByteEncoder<ArrayType>:
## a single-byte encoder over an index of the code points of bytes 0x80 to 0xFF.
pub func single_byte_encoder_construct(this: SingleByteEncoder[u16]*, translation_table: u16[128]):
this.encoder_base.vtable = &single_byte_encoder_vtable.encoder_vtable # the C++ vptr
this.encoder_base.class_id = single_byte_encoder_class.first_id
this.m_translation_table = translation_table
## Ported from Libraries/LibTextCodec/Encoder.cpp:642 TextCodec::SingleByteEncoder::process.
## https://encoding.spec.whatwg.org/#single-byte-encoder
pub func single_byte_encoder_process(this: SingleByteEncoder[u16]*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
# for (u32 const code_point : input)
let end = ak.utf8_view_end(&input)
var iterator = ak.utf8_view_begin(&input)
while iterator != end:
let code_point = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
if code_point < 0x80:
# 2. If code point is an ASCII code point, return a byte whose value is code point.
ak.function1_call(&on_byte, (u8)code_point)
else:
var pointer: u8? = none
for i: u8 in 0..<128:
if this.m_translation_table[i] == code_point:
# 3. Let pointer be the index pointer for code point in index single-byte.
pointer = i
break
if let index_pointer = pointer:
# 5. Return a byte whose value is pointer + 0x80.
ak.function1_call(&on_byte, index_pointer + 0x80)
else:
# 4. If pointer is null, return error with code point.
ak.function1_call(&on_error, code_point)
# 1. If code point is end-of-queue, return finished.
return
## The vtable slot of SingleByteEncoder<u16>::process (override): dispatches to
## single_byte_encoder_process.
pub func single_byte_encoder_process_impl(base: Encoder*, input: ak.Utf8View, on_byte: ak.Function1[u8, unit], on_error: ak.Function1[u32, unit]) -> !:
try single_byte_encoder_process((SingleByteEncoder[u16]*)base, input, on_byte, on_error)
# mark: Isomorphic encode ======================================================================
## Ported from Libraries/LibTextCodec/Encoder.cpp:671 TextCodec::isomorphic_encode: the bytes whose
## values are the input's code points (encoding as ISO-8859-1 / Latin-1).
## https://infra.spec.whatwg.org/#isomorphic-encode
pub func isomorphic_encode(input: ak.StringView) -> ak.ByteString:
# To isomorphic encode an isomorphic string input: return a byte sequence whose length is equal to input’s code
# point length and whose bytes have the same values as the values of input’s code points, in the same order.
# NB: This is essentially spec-speak for "Encode as ISO-8859-1 / Latin-1".
var builder: ak.StringBuilder
ak.string_builder_construct_usize(&builder, ak.string_view_length(&input))
# for (auto code_point : Utf8View { input })
var view: ak.Utf8View
ak.utf8_view_construct_string_view(&view, input)
let end = ak.utf8_view_end(&view)
var iterator = ak.utf8_view_begin(&view)
while iterator != end:
let code_point = ak.utf8_code_point_iterator_deref(&iterator)
_ = ak.utf8_code_point_iterator_inc(&iterator)
# VERIFY(code_point <= 0xFF);
if code_point > 0xFF:
ak.dbgln("FIXME: Trying to isomorphic encode a string with code points > U+00FF.")
ak.string_builder_append_u8(&builder, (u8)code_point)
return ak.string_builder_to_byte_string(&builder)