Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-mime

Source of release 0.6.0

0.6.0 / src / mime / html.lucb

#==============================================================================================
#
#   mime/html - HTML made into readable text
#
#   DESCRIPTION:
#       Until the window shows HTML mail through the browser engine, an HTML body is
#       read as text: tags give structure (paragraphs and headings a blank line, line
#       breaks and divisions a new line, list items a bullet or a number, table cells a
#       gap, block quotes a `> ` prefix, images their alt text in brackets), runs of
#       whitespace collapse except inside <pre>, character references are decoded, and
#       what is never shown (head, style, script, comments) is skipped. Links show their
#       text only. The input is already UTF-8.
#
#==============================================================================================

## Named references outside Latin-1 that mail uses, with their scalars.
let named_references: (str, u32)[48] = [
    ("amp", 38), ("lt", 60), ("gt", 62), ("quot", 34), ("apos", 39), ("hellip", 0x2026),
    ("mdash", 0x2014), ("ndash", 0x2013), ("lsquo", 0x2018), ("rsquo", 0x2019), ("sbquo", 0x201A),
    ("ldquo", 0x201C), ("rdquo", 0x201D), ("bdquo", 0x201E), ("bull", 0x2022), ("euro", 0x20AC),
    ("trade", 0x2122), ("zwnj", 0x200C), ("zwj", 0x200D), ("lrm", 0x200E), ("rlm", 0x200F),
    ("thinsp", 0x2009), ("ensp", 0x2002), ("emsp", 0x2003), ("larr", 0x2190), ("rarr", 0x2192),
    ("uarr", 0x2191), ("darr", 0x2193), ("harr", 0x2194), ("dagger", 0x2020), ("Dagger", 0x2021),
    ("permil", 0x2030), ("prime", 0x2032), ("Prime", 0x2033), ("lsaquo", 0x2039), ("rsaquo", 0x203A),
    ("oline", 0x203E), ("OElig", 0x0152), ("oelig", 0x0153), ("Scaron", 0x0160), ("scaron", 0x0161),
    ("Yuml", 0x0178), ("fnof", 0x0192), ("circ", 0x02C6), ("tilde", 0x02DC), ("check", 0x2713),
    ("star", 0x2606), ("hearts", 0x2665),
]

## The names of U+00A0-U+00FF, in order.
let latin1_names: str[96] = [
    "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect",
    "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr",
    "deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot",
    "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest",
    "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil",
    "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml",
    "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times",
    "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig",
    "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil",
    "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml",
    "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide",
    "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml",
]

## Elements that end the line they are on.
let line_elements: str[16] = ["br", "div", "tr", "li", "dt", "dd", "address", "center", "section",
                              "article", "header", "footer", "nav", "form", "caption", "tbody"]
## Elements set off by a blank line.
let block_elements: str[15] = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "table",
                               "ul", "ol", "dl", "pre", "hr", "figure"]
## Elements whose content is never shown.
let hidden_elements: str[6] = ["head", "style", "script", "title", "template", "xml"]

## The scalar a character reference names (the text between `&` and `;`), or none.
func reference_scalar(name: const u8[]) -> u32?:
    if name.length > 1 and name[0] == '#':
        var value: u32 = 0
        let hexadecimal = name[1] == 'x' or name[1] == 'X'
        let digits = name[2..] if hexadecimal else name[1..]
        if digits.length == 0 or digits.length > 7:
            return none
        for byte in digits:
            if hexadecimal:
                let digit = hex_value(byte) else return none
                value = value * 16 + digit
            else:
                if byte < '0' or byte > '9':
                    return none
                value = value * 10 + (byte - '0')
        # Control references map as browsers map them; zero is the replacement character.
        if value == 0:
            return 0xFFFD
        if value >= 0x80 and value < 0xA0:
            return windows1252_high[value - 0x80]
        return value
    for entry in named_references:
        if name == entry.0.bytes:
            return entry.1
    for at in 0..<latin1_names.length:
        if name == latin1_names[at].bytes:
            return 0xA0 + (u32)at
    return none

## Whether `name` is one of `names`, ignoring case.
func is_one_of(name: const u8[], names: const str[]) -> bool:
    for candidate in names:
        if same(name, candidate.bytes):
            return true
    return false

## The text being written: whitespace collapsing, line breaks, quote prefixes.
struct TextWriter:
    var out: Bytes*
    ## Line breaks at the end of what was written, at most two.
    var breaks: u32
    var space: bool
    var started: bool
    var quote_depth: u32
    var pre_depth: u32

    ## Begin a line: the quote prefix.
    func begin_line() -> !:
        var level: u32 = 0
        while level < self.quote_depth:
            try self.out.put(b"> ")
            level += 1

    ## Write visible text, collapsing whitespace outside <pre>.
    func text(data: const u8[]) -> !:
        for byte in data:
            if self.pre_depth > 0 and byte == '\n':
                try self.out.byte('\n')
                self.breaks = 1
                self.started = true
                continue
            if self.pre_depth == 0 and is_space(byte):
                self.space = true
                continue
            if self.breaks > 0 or not self.started:
                try self.begin_line()
            elif self.space:
                try self.out.byte(' ')
            self.space = false
            self.breaks = 0
            self.started = true
            try self.out.byte(byte)

    ## Write one scalar as visible text.
    func scalar(scalar: u32) -> !:
        if scalar == 0xA0:
            try self.text(b"\xC2\xA0")
            return
        var buffer: u8[4]
        let count = utf8.encode(scalar, buffer) else 0
        if count == 0:
            try self.text(b"\xEF\xBF\xBD")
        else:
            try self.text(buffer[0..<count])

    ## A line break, as <br> makes: two in a row leave a blank line, more add nothing.
    func line_break() -> !:
        self.space = false
        if self.started and self.breaks < 2:
            try self.out.byte('\n')
            self.breaks += 1

    ## End the line, then leave `count` line breaks in all (a blank line for two).
    func breaks_to(count: u32) -> !:
        self.space = false
        if not self.started:
            return
        while self.breaks < count:
            try self.out.byte('\n')
            self.breaks += 1

## The tag starting at `start` (at `<`): its name, whether it closes, and where it ends.
struct Tag:
    var name: const u8[]
    var closing: bool
    var start: usize
    var end: usize

## Read the tag at `start`; none when `<` does not begin one.
func read_tag(data: const u8[], start: usize) -> Tag?:
    var index = start + 1
    var closing = false
    if index < data.length and data[index] == '/':
        closing = true
        index += 1
    let name_start = index
    while index < data.length and ((data[index] >= 'a' and data[index] <= 'z') or (data[index] >= 'A' and data[index] <= 'Z') or (data[index] >= '0' and data[index] <= '9')):
        index += 1
    if index == name_start:
        return none
    let name = data[name_start..<index]
    # Attributes run to the `>` that is outside quotes.
    var quote: u8 = 0
    while index < data.length:
        let byte = data[index]
        if quote != 0:
            if byte == quote:
                quote = 0
        elif byte == '"' or byte == '\'':
            quote = byte
        elif byte == '>':
            return Tag(name = name, closing = closing, start = start, end = index + 1)
        index += 1
    return Tag(name = name, closing = closing, start = start, end = data.length)

## The value of attribute `name` inside tag text `tag`, or none.
func attribute_value(tag: const u8[], name: str) -> const u8[]?:
    var index: usize = 1
    while index + name.length < tag.length:
        if is_space(tag[index - 1]) and starts(tag[index..], name.bytes):
            var at = index + name.length
            while at < tag.length and is_space(tag[at]):
                at += 1
            if at < tag.length and tag[at] == '=':
                at += 1
                while at < tag.length and is_space(tag[at]):
                    at += 1
                if at < tag.length and (tag[at] == '"' or tag[at] == '\''):
                    let quote = tag[at]
                    let close = index_of(tag, quote, at + 1) else tag.length
                    return tag[at + 1..<close]
                var stop = at
                while stop < tag.length and not is_space(tag[stop]) and tag[stop] != '>':
                    stop += 1
                return tag[at..<stop]
        index += 1
    return none

## HTML being read: the text writer and the list being numbered.
struct HtmlReader:
    var writer: TextWriter
    var ordered: u32
    var in_ordered: bool

    ## Act on `tag` from `data`; where reading continues.
    func tag(data: const u8[], tag: Tag) -> usize!:
        let name = tag.name
        if not tag.closing and is_one_of(name, hidden_elements):
            # Skip to the matching close tag.
            var at = tag.end
            while at < data.length:
                if data[at] == '<' and at + 2 + name.length <= data.length and data[at + 1] == '/' and same(data[at + 2..<at + 2 + name.length], name):
                    return (index_of(data, '>', at) else (data.length - 1)) + 1
                at += 1
            return data.length
        if same(name, b"br"):
            try self.writer.line_break()
        elif same(name, b"blockquote"):
            try self.writer.breaks_to(2)
            if not tag.closing:
                self.writer.quote_depth += 1
            elif self.writer.quote_depth > 0:
                self.writer.quote_depth -= 1
        elif same(name, b"pre"):
            try self.writer.breaks_to(2)
            if not tag.closing:
                self.writer.pre_depth += 1
            elif self.writer.pre_depth > 0:
                self.writer.pre_depth -= 1
        elif same(name, b"ol") or same(name, b"ul"):
            self.in_ordered = same(name, b"ol") and not tag.closing
            self.ordered = 0
            try self.writer.breaks_to(2 if tag.closing else 1)
        elif same(name, b"li"):
            if not tag.closing:
                try self.writer.breaks_to(1)
                if self.in_ordered:
                    self.ordered += 1
                    var number: Bytes
                    defer number.close()
                    try put_number((i64)self.ordered, 1, &number)
                    try number.byte('.')
                    try self.writer.text(number.view())
                else:
                    try self.writer.text("•".bytes)
                # The space after the marker, written before the item's first word.
                self.writer.space = true
        elif same(name, b"hr"):
            try self.writer.breaks_to(2)
            try self.writer.text(b"----------")
            try self.writer.breaks_to(2)
        elif same(name, b"td") or same(name, b"th"):
            if not tag.closing and self.writer.breaks == 0 and self.writer.started:
                try self.writer.out.put(b"  ")
        elif same(name, b"img"):
            if let alt = attribute_value(data[tag.start..<tag.end], "alt"):
                if trim(alt).length > 0:
                    try self.writer.text(b" [")
                    try self.writer.text(trim(alt))
                    try self.writer.text(b"] ")
        elif is_one_of(name, block_elements):
            try self.writer.breaks_to(2)
        elif is_one_of(name, line_elements):
            try self.writer.breaks_to(1)
        return tag.end

## The offset just past the comment, declaration or processing instruction at `start`.
func skip_markup(data: const u8[], start: usize) -> usize:
    if starts(data[start..], b"<!--"):
        var at = start + 4
        while at + 2 < data.length:
            if data[at] == '-' and data[at + 1] == '-' and data[at + 2] == '>':
                return at + 3
            at += 1
        return data.length
    return (index_of(data, '>', start) else (data.length - 1)) + 1

## Append HTML `data` (UTF-8) as readable text.
pub func html_to_text(data: const u8[], out: Bytes*) -> !:
    var reader = HtmlReader(writer = TextWriter(out = out))
    var index: usize = 0
    while index < data.length:
        let byte = data[index]
        if byte == '<':
            if index + 1 < data.length and (data[index + 1] == '!' or data[index + 1] == '?'):
                index = skip_markup(data, index)
            elif let tag = read_tag(data, index):
                index = try reader.tag(data, tag)
            else:
                try reader.writer.text(b"<")
                index += 1
            continue
        if byte == '&':
            var end = index + 1
            while end < data.length and end - index < 12 and data[end] != ';' and not is_space(data[end]) and data[end] != '&' and data[end] != '<':
                end += 1
            if end < data.length and data[end] == ';':
                if let scalar = reference_scalar(data[index + 1..<end]):
                    try reader.writer.scalar(scalar)
                    index = end + 1
                    continue
            try reader.writer.text(b"&")
            index += 1
            continue
        var end = index + 1
        while end < data.length and data[end] != '<' and data[end] != '&':
            end += 1
        try reader.writer.text(data[index..<end])
        index = end