dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / src / text_codec / decoder_1.lucb
#==============================================================================================
#
# decoder_1 - the decoder instances, decoder lookup by label and BOM sniffing
#
# DESCRIPTION:
# Port of Libraries/LibTextCodec/Decoder.cpp (lines 1-337): one decoder instance per
# encoding, decoder_for and decoder_for_exact_name, the Encoding Standard's label table
# (get_standardized_encoding), BOM sniffing, "decode" with a fallback decoder, and
# get_output_encoding.
#
#==============================================================================================
# mark: Decoder instances ======================================================================
# The C++ globals of Decoder.cpp's anonymous namespace. Their constructors only set the vtable
# pointer (and a single-byte decoder's table), so each is a constant-initialized global here; the
# class id is the class's first id in types_decoder.
var s_latin1_decoder: Latin1Decoder = Latin1Decoder(decoder_base = Decoder(vtable = &latin1_decoder_vtable.decoder_vtable, class_id = 6))
var s_utf8_decoder: Utf8Decoder = Utf8Decoder(decoder_base = Decoder(vtable = &utf8_decoder_vtable.decoder_vtable, class_id = 12))
var s_utf16be_decoder: Utf16BeDecoder = Utf16BeDecoder(decoder_base = Decoder(vtable = &utf16_be_decoder_vtable.decoder_vtable, class_id = 10))
var s_utf16le_decoder: Utf16LeDecoder = Utf16LeDecoder(decoder_base = Decoder(vtable = &utf16_le_decoder_vtable.decoder_vtable, class_id = 11))
var s_pdf_doc_encoding_decoder: PdfDocEncodingDecoder = PdfDocEncodingDecoder(decoder_base = Decoder(vtable = &pdf_doc_encoding_decoder_vtable.decoder_vtable, class_id = 7))
var s_x_user_defined_decoder: XUserDefinedDecoder = XUserDefinedDecoder(decoder_base = Decoder(vtable = &x_user_defined_decoder_vtable.decoder_vtable, class_id = 13))
var s_gb18030_decoder: Gb18030Decoder = Gb18030Decoder(decoder_base = Decoder(vtable = &gb18030_decoder_vtable.decoder_vtable, class_id = 4))
var s_big5_decoder: Big5Decoder = Big5Decoder(decoder_base = Decoder(vtable = &big5_decoder_vtable.decoder_vtable, class_id = 1))
var s_euc_jp_decoder: EucjpDecoder = EucjpDecoder(decoder_base = Decoder(vtable = &eucjp_decoder_vtable.decoder_vtable, class_id = 2))
var s_iso_2022_jp_decoder: Iso2022JpDecoder = Iso2022JpDecoder(decoder_base = Decoder(vtable = &iso2022_jp_decoder_vtable.decoder_vtable, class_id = 5))
var s_shift_jis_decoder: ShiftJisDecoder = ShiftJisDecoder(decoder_base = Decoder(vtable = &shift_jis_decoder_vtable.decoder_vtable, class_id = 9))
var s_euc_kr_decoder: EuckrDecoder = EuckrDecoder(decoder_base = Decoder(vtable = &euckr_decoder_vtable.decoder_vtable, class_id = 3))
var s_replacement_decoder: ReplacementDecoder = ReplacementDecoder(decoder_base = Decoder(vtable = &replacement_decoder_vtable.decoder_vtable, class_id = 8))
# s_{encoding}_index is generated from https://encoding.spec.whatwg.org/indexes.json
# Found separately in https://encoding.spec.whatwg.org/index-{encoding}.txt
var s_ibm866_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_ibm866_index)
var s_latin2_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_2_index)
var s_latin3_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_3_index)
var s_latin4_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_4_index)
var s_latin_cyrillic_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_5_index)
var s_latin_arabic_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_6_index)
var s_latin_greek_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_7_index)
var s_latin_hebrew_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_8_index)
var s_latin6_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_10_index)
var s_latin7_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_13_index)
var s_latin8_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_14_index)
var s_latin9_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_15_index)
var s_latin10_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_iso_8859_16_index)
var s_centraleurope_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1250_index)
var s_cyrillic_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1251_index)
var s_hebrew_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1255_index)
var s_koi8r_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_koi8_r_index)
var s_koi8u_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_koi8_u_index)
var s_mac_roman_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_macintosh_index)
var s_windows874_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_874_index)
var s_windows1252_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1252_index)
var s_windows1253_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1253_index)
var s_turkish_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1254_index)
var s_windows1256_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1256_index)
var s_windows1257_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1257_index)
var s_windows1258_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_windows_1258_index)
var s_mac_cyrillic_decoder: SingleByteDecoder[u16] = SingleByteDecoder[u16](decoder_base = Decoder(vtable = &single_byte_decoder_vtable.decoder_vtable, class_id = 14), m_translation_table = s_x_mac_cyrillic_index)
# mark: Decoder lookup =========================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:71 TextCodec::decoder_for: the decoder of the
## encoding a label names (https://encoding.spec.whatwg.org/#concept-encoding-get), or none.
pub func decoder_for(label: ak.StringView) -> Decoder*?:
if let encoding = get_standardized_encoding(label):
return decoder_for_exact_name(encoding)
return none
## Ported from Libraries/LibTextCodec/Decoder.cpp:77 TextCodec::decoder_for_exact_name: the
## decoder for the exact name given, skipping get_standardized_encoding. Use this when you want
## ISO-8859-1 instead of windows-1252.
pub func decoder_for_exact_name(encoding: ak.StringView) -> Decoder*?:
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-1")):
return s_latin1_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1252")):
return s_windows1252_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("utf-8")):
return s_utf8_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("utf-16be")):
return s_utf16be_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("utf-16le")):
return s_utf16le_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("big5")):
return s_big5_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("euc-jp")):
return s_euc_jp_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("euc-kr")):
return s_euc_kr_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("gbk")):
return s_gb18030_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("gb18030")):
return s_gb18030_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("ibm866")):
return s_ibm866_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-2022-jp")):
return s_iso_2022_jp_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-2")):
return s_latin2_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-3")):
return s_latin3_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-4")):
return s_latin4_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-5")):
return s_latin_cyrillic_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-6")):
return s_latin_arabic_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-7")):
return s_latin_greek_decoder.decoder()
if ak.string_view_is_one_of_ignoring_ascii_case(&encoding, ["iso-8859-8", "iso-8859-8-i"]):
return s_latin_hebrew_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-10")):
return s_latin6_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-13")):
return s_latin7_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-14")):
return s_latin8_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-15")):
return s_latin9_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("iso-8859-16")):
return s_latin10_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("koi8-r")):
return s_koi8r_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("koi8-u")):
return s_koi8u_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("macintosh")):
return s_mac_roman_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("PDFDocEncoding")):
return s_pdf_doc_encoding_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("replacement")):
return s_replacement_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("shift_jis")):
return s_shift_jis_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-874")):
return s_windows874_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1250")):
return s_centraleurope_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1251")):
return s_cyrillic_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1253")):
return s_windows1253_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1254")):
return s_turkish_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1255")):
return s_hebrew_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1256")):
return s_windows1256_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1257")):
return s_windows1257_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("windows-1258")):
return s_windows1258_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("x-mac-cyrillic")):
return s_mac_cyrillic_decoder.decoder()
if ak.string_view_equals_ignoring_ascii_case(&encoding, ak.sv("x-user-defined")):
return s_x_user_defined_decoder.decoder()
ak.dbgln(f"TextCodec: No decoder implemented for encoding '{ak.string_view_text(encoding)}'")
return none
# mark: The label table =========================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:166 TextCodec::get_standardized_encoding: the
## name of the encoding a label names, or none.
## https://encoding.spec.whatwg.org/#concept-encoding-get
pub func get_standardized_encoding(encoding: ak.StringView) -> ak.StringView?:
# 1. Remove any leading and trailing ASCII whitespace from label.
# https://infra.spec.whatwg.org/#ascii-whitespace: ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 SPACE.
let label = ak.string_view_trim(&encoding, ak.sv("\t\n\u{c}\r "))
# 2. If label is an ASCII case-insensitive match for any of the labels listed in the table below, then return the corresponding encoding; otherwise return failure.
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["unicode-1-1-utf-8", "unicode11utf8", "unicode20utf8", "utf-8", "utf8", "x-unicode20utf8"]):
return ak.sv("UTF-8")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["866", "cp866", "csibm866", "ibm866"]):
return ak.sv("IBM866")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatin2", "iso-8859-2", "iso-ir-101", "iso8859-2", "iso88592", "iso_8859-2",
"iso_8859-2:1987", "l2", "latin2"]):
return ak.sv("ISO-8859-2")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatin3", "iso-8859-3", "iso-ir-109", "iso8859-3", "iso88593", "iso_8859-3",
"iso_8859-3:1988", "l3", "latin3"]):
return ak.sv("ISO-8859-3")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatin4", "iso-8859-4", "iso-ir-110", "iso8859-4", "iso88594", "iso_8859-4",
"iso_8859-4:1988", "l4", "latin4"]):
return ak.sv("ISO-8859-4")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatincyrillic", "cyrillic", "iso-8859-5", "iso-ir-144", "iso8859-5", "iso88595",
"iso_8859-5", "iso_8859-5:1988"]):
return ak.sv("ISO-8859-5")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"arabic", "asmo-708", "csiso88596e", "csiso88596i", "csisolatinarabic", "ecma-114",
"iso-8859-6", "iso-8859-6-e", "iso-8859-6-i", "iso-ir-127", "iso8859-6", "iso88596",
"iso_8859-6", "iso_8859-6:1987"]):
return ak.sv("ISO-8859-6")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatingreek", "ecma-118", "elot_928", "greek", "greek8", "iso-8859-7",
"iso-ir-126", "iso8859-7", "iso88597", "iso_8859-7", "iso_8859-7:1987", "sun_eu_greek"]):
return ak.sv("ISO-8859-7")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csiso88598e", "csisolatinhebrew", "hebrew", "iso-8859-8", "iso-8859-8-e", "iso-ir-138",
"iso8859-8", "iso88598", "iso_8859-8", "iso_8859-8:1988", "visual"]):
return ak.sv("ISO-8859-8")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["csiso88598i", "iso-8859-8-i", "logical"]):
return ak.sv("ISO-8859-8-I")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatin6", "iso-8859-10", "iso-ir-157", "iso8859-10", "iso885910", "l6", "latin6"]):
return ak.sv("ISO-8859-10")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["iso-8859-13", "iso8859-13", "iso885913"]):
return ak.sv("ISO-8859-13")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["iso-8859-14", "iso8859-14", "iso885914"]):
return ak.sv("ISO-8859-14")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csisolatin9", "iso-8859-15", "iso8859-15", "iso885915", "iso_8859-15", "l9"]):
return ak.sv("ISO-8859-15")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["iso-8859-16"]):
return ak.sv("ISO-8859-16")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cskoi8r", "koi", "koi8", "koi8-r", "koi8_r"]):
return ak.sv("KOI8-R")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["koi8-ru", "koi8-u"]):
return ak.sv("KOI8-U")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["csmacintosh", "mac", "macintosh", "x-mac-roman"]):
return ak.sv("macintosh")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["pdfdocencoding"]):
return ak.sv("PDFDocEncoding")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"dos-874", "iso-8859-11", "iso8859-11", "iso885911", "tis-620", "windows-874"]):
return ak.sv("windows-874")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1250", "windows-1250", "x-cp1250"]):
return ak.sv("windows-1250")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1251", "windows-1251", "x-cp1251"]):
return ak.sv("windows-1251")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"ansi_x3.4-1968", "ascii", "cp1252", "cp819", "csisolatin1", "ibm819", "iso-8859-1",
"iso-ir-100", "iso8859-1", "iso88591", "iso_8859-1", "iso_8859-1:1987", "l1", "latin1",
"us-ascii", "windows-1252", "x-cp1252"]):
return ak.sv("windows-1252")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1253", "windows-1253", "x-cp1253"]):
return ak.sv("windows-1253")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"cp1254", "csisolatin5", "iso-8859-9", "iso-ir-148", "iso8859-9", "iso88599",
"iso_8859-9", "iso_8859-9:1989", "l5", "latin5", "windows-1254", "x-cp1254"]):
return ak.sv("windows-1254")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1255", "windows-1255", "x-cp1255"]):
return ak.sv("windows-1255")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1256", "windows-1256", "x-cp1256"]):
return ak.sv("windows-1256")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1257", "windows-1257", "x-cp1257"]):
return ak.sv("windows-1257")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cp1258", "windows-1258", "x-cp1258"]):
return ak.sv("windows-1258")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["x-mac-cyrillic", "x-mac-ukrainian"]):
return ak.sv("x-mac-cyrillic")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["koi8-r", "koi8r"]):
return ak.sv("koi8-r")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"chinese", "csgb2312", "csiso58gb231280", "gb2312", "gb_2312", "gb_2312-80", "gbk",
"iso-ir-58", "x-gbk"]):
return ak.sv("GBK")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["gb18030"]):
return ak.sv("gb18030")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["big5", "big5-hkscs", "cn-big5", "csbig5", "x-x-big5"]):
return ak.sv("Big5")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["cseucpkdfmtjapanese", "euc-jp", "x-euc-jp"]):
return ak.sv("EUC-JP")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["csiso2022jp", "iso-2022-jp"]):
return ak.sv("ISO-2022-JP")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csshiftjis", "ms932", "ms_kanji", "shift-jis", "shift_jis", "sjis", "windows-31j",
"x-sjis"]):
return ak.sv("Shift_JIS")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"cseuckr", "csksc56011987", "euc-kr", "iso-ir-149", "korean", "ks_c_5601-1987",
"ks_c_5601-1989", "ksc5601", "ksc_5601", "windows-949"]):
return ak.sv("EUC-KR")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csiso2022kr", "hz-gb-2312", "iso-2022-cn", "iso-2022-cn-ext", "iso-2022-kr",
"replacement"]):
return ak.sv("replacement")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["unicodefffe", "utf-16be"]):
return ak.sv("UTF-16BE")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, [
"csunicode", "iso-10646-ucs-2", "ucs-2", "unicode", "unicodefeff", "utf-16", "utf-16le"]):
return ak.sv("UTF-16LE")
if ak.string_view_is_one_of_ignoring_ascii_case(&label, ["x-user-defined"]):
return ak.sv("x-user-defined")
ak.dbgln(f"TextCodec: Unrecognized encoding: {ak.string_view_text(label)}")
return none
# mark: BOM sniffing and decode ================================================================
## Ported from Libraries/LibTextCodec/Decoder.cpp:263 TextCodec::bom_sniff_to_decoder: the Unicode
## decoder for the byte order mark the input starts with, or none.
## https://encoding.spec.whatwg.org/#bom-sniff
pub func bom_sniff_to_decoder(input: ak.StringView) -> Decoder*?:
# 1. Let BOM be the result of peeking 3 bytes from ioQueue, converted to a byte sequence.
# 2. For each of the rows in the table below, starting with the first one and going down,
# if BOM starts with the bytes given in the first column, then return the encoding given
# in the cell in the second column of that row. Otherwise, return null.
# Byte Order Mark | Encoding
# --------------------------
# 0xEF 0xBB 0xBF | UTF-8
# 0xFE 0xFF | UTF-16BE
# 0xFF 0xFE | UTF-16LE
let bytes = ak.string_view_bytes(&input)
if bytes.length < 2:
return none
let first_byte = bytes[0]
match first_byte:
0xEF:
# UTF-8
if bytes.length < 3:
return none
if bytes[1] == 0xBB and bytes[2] == 0xBF:
return s_utf8_decoder.decoder()
return none
0xFE:
# UTF-16BE
if bytes[1] == 0xFF:
return s_utf16be_decoder.decoder()
return none
0xFF:
# UTF-16LE
if bytes[1] == 0xFE:
return s_utf16le_decoder.decoder()
return none
_:
return none # past the switch: return {}
## Ported from Libraries/LibTextCodec/Decoder.cpp:303
## TextCodec::convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte_order_mark: decodes
## with the given decoder unless the input starts with a byte order mark, in which case with the
## matching Unicode decoder. The name is obnoxious on purpose: use it only when you must (XHR
## does).
## https://encoding.spec.whatwg.org/#decode
pub func convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte_order_mark(fallback_decoder: Decoder*, input: ak.StringView) -> ak.String!:
var actual_decoder = fallback_decoder
var remaining_input = input # C++ reassigns the `input` parameter
# 1. Let BOMEncoding be the result of BOM sniffing ioQueue.
# 2. If BOMEncoding is non-null:
if let unicode_decoder = bom_sniff_to_decoder(input):
# 1. Set encoding to BOMEncoding.
actual_decoder = unicode_decoder
# 2. Read three bytes from ioQueue, if BOMEncoding is UTF-8; otherwise read two bytes. (Do nothing with those bytes.)
# FIXME: I imagine this will be pretty slow for large inputs, as it's regenerating the input without the first 2/3 bytes.
remaining_input = ak.string_view_substring_view(&input, 3 if unicode_decoder == s_utf8_decoder.decoder() else 2)
# VERIFY(actual_decoder): a Decoder* is never null.
# 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
# FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
let output = try decoder_to_utf8(actual_decoder, remaining_input)
# 4. Return output.
return output
## Ported from Libraries/LibTextCodec/Decoder.cpp:329 TextCodec::get_output_encoding: the encoding
## to encode output in when a document's encoding is the given one.
## https://encoding.spec.whatwg.org/#get-an-output-encoding
pub func get_output_encoding(encoding: ak.StringView) -> ak.StringView:
# 1. If encoding is replacement or UTF-16BE/LE, then return UTF-8.
if ak.string_view_is_one_of_ignoring_ascii_case(&encoding, ["replacement", "utf-16le", "utf-16be"]):
return ak.sv("UTF-8")
# 2. Return encoding.
return encoding