Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-mime

Source of release 0.6.0

0.6.0 / src / mime / charsets.lucb

#==============================================================================================
#
#   mime/charsets - Bytes in a named charset, as UTF-8
#
#   DESCRIPTION:
#       Mail names its charset per part and per encoded word. Built in are the ones that
#       carry nearly all of it: UTF-8, US-ASCII, ISO-8859-1, ISO-8859-15 and Windows-1252
#       (ISO-8859-1 and US-ASCII decode as Windows-1252, as WHATWG has browsers do, since
#       senders mislabel it). An application adds the rest (Shift_JIS, GBK, KOI8-R, …) by
#       installing a `CharsetDecoder`, for example one over luce-browser-foundation's
#       text_codec. A charset nobody knows decodes as UTF-8 when the bytes are valid UTF-8,
#       and as Windows-1252 otherwise. Invalid UTF-8 becomes U+FFFD; nothing fails.
#
#==============================================================================================

## Decodes `data` in `charset` (lower case, trimmed) onto the output as UTF-8, answering
## whether it knew the charset.
pub type CharsetDecoder = func(str, const u8[], Bytes*) -> bool!

## The decoder for charsets this package does not know; none until an application sets one.
var extra_decoder: CharsetDecoder?

## Install `decoder` for the charsets the built-in ones do not cover. Set it once, before
## messages are decoded on any thread.
pub func set_charset_decoder(decoder: CharsetDecoder?):
    extra_decoder = decoder

## The charsets decoded here.
enum Builtin:
    utf8
    windows1252
    latin9

## Unicode for Windows-1252's 0x80-0x9F; the five undefined bytes keep their C1 value.
let windows1252_high: u16[32] = [
    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
    0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
    0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,
]

## The built-in decoder named by `name`, or none.
func builtin(name: const u8[]) -> Builtin?:
    for alias in ["utf-8", "utf8", "unicode-1-1-utf-8"]:
        if same(name, alias.bytes):
            return Builtin.utf8
    for alias in ["windows-1252", "cp1252", "x-cp1252", "iso-8859-1", "iso8859-1", "iso_8859-1",
                  "latin1", "latin-1", "l1", "cp819", "us-ascii", "ascii", "ansi_x3.4-1968", "us"]:
        if same(name, alias.bytes):
            return Builtin.windows1252
    for alias in ["iso-8859-15", "iso8859-15", "iso_8859-15", "latin-9", "latin9", "l9"]:
        if same(name, alias.bytes):
            return Builtin.latin9
    return none

## The Unicode scalar of byte `byte` in ISO-8859-15.
func latin9_scalar(byte: u8) -> u32:
    return match byte:
        0xA4 => 0x20AC
        0xA6 => 0x0160
        0xA8 => 0x0161
        0xB4 => 0x017D
        0xB8 => 0x017E
        0xBC => 0x0152
        0xBD => 0x0153
        0xBE => 0x0178
        _ => byte

## Append `data`, taken as UTF-8, with each invalid sequence as U+FFFD.
pub func put_utf8(data: const u8[], out: Bytes*) -> !:
    var index: usize = 0
    while index < data.length:
        if data[index] < 0x80:
            try out.byte(data[index])
            index += 1
            continue
        if let decoded = utf8.decode(data, index):
            try out.put(data[index..<index + decoded.width])
            index += decoded.width
        else:
            try out.put(b"\xEF\xBF\xBD")
            index += 1

## Append `data`, taken as Windows-1252, or as ISO-8859-15 with `latin9`.
func put_single_byte(data: const u8[], out: Bytes*, latin9: bool) -> !:
    for byte in data:
        if byte < 0x80:
            try out.byte(byte)
        elif latin9:
            try out.scalar(latin9_scalar(byte))
        elif byte < 0xA0:
            try out.scalar(windows1252_high[byte - 0x80])
        else:
            try out.scalar(byte)

## Append `data`, in the charset called `charset`, as UTF-8.
pub func decode_charset(charset: const u8[], data: const u8[], out: Bytes*) -> !:
    let name = trim(charset)
    if let known = builtin(name):
        match known:
            .utf8: try put_utf8(data, out)
            .windows1252: try put_single_byte(data, out, false)
            .latin9: try put_single_byte(data, out, true)
        return
    # ISO-2022-JP and UTF-7 are 7-bit, so even ASCII bytes go to the decoder first.
    if let decoder = extra_decoder:
        var lowered: u8[64]
        if name.length <= lowered.length and is_ascii(name):
            for index in 0..<name.length:
                lowered[index] = lower(name[index])
            if try decoder((str)lowered[0..<name.length], data, out):
                return
    if utf8.valid(data):
        try out.put(data)
    else:
        try put_single_byte(data, out, false)