Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-prism

Source of release 0.6.0

0.6.0 / src / codecs / html_reader.lucb

## HTML imports into the shared structured-document vocabulary.
import codecs.xml
from codecs.common import append, blank, whitespace, starts, set_text, set_integer, set_boolean
from model import DocumentData
from storage import Buffer
from types import invalid, limit_exceeded

func raw_tag(name: str) -> bool:
    return name == "svg" or name == "math"

func inline_tag(name: str) -> bool:
    return name == "a" or name == "em" or name == "i" or name == "strong" or name == "b" or name == "code" or name == "br" or name == "img" or name == "span"

func class_contains(token: const xml.Token*, name: str) -> bool:
    let text = token.attr("class") else return false
    var i: usize = 0
    while i < text.length:
        while i < text.length and whitespace(text.bytes[i]):
            i += 1
        let start = i
        while i < text.length and not whitespace(text.bytes[i]):
            i += 1
        if (str)text.bytes[start..<i] == name:
            return true
    return false

struct Parser:
    var source: str
    var tokens: const xml.Tokens*
    var doc: DocumentData*
    var position: usize

    func at_close(name: str) -> bool:
        if self.position == self.tokens.values.count:
            return false
        let token = &self.tokens.values.items[self.position]
        return token.kind == xml.Kind.close and token.name.view() == name

    func inline_content(parent: str, close: str, depth: usize) -> !:
        if depth > 128:
            error(limit_exceeded, "HTML inline nesting exceeds 128")
        while self.position < self.tokens.values.count:
            let token = &self.tokens.values.items[self.position]
            let name = token.name.view()
            if token.kind == xml.Kind.close:
                if name == close:
                    self.position += 1
                return
            if token.kind == xml.Kind.text:
                if token.text.used > 0:
                    var child = try append(self.doc, parent, "text")
                    defer child.close()
                    try set_text(self.doc, child.view(), "text", token.text.view())
                self.position += 1
            elif token.kind == xml.Kind.empty:
                if name == "br":
                    var child = try append(self.doc, parent, "linebreak")
                    child.close()
                elif name == "img":
                    var child = try append(self.doc, parent, "image")
                    defer child.close()
                    if let src = token.attr("src"):
                        try set_text(self.doc, child.view(), "src", src)
                    if let alt = token.attr("alt"):
                        try set_text(self.doc, child.view(), "alt", alt)
                self.position += 1
            elif name == "em" or name == "i" or name == "strong" or name == "b" or name == "a":
                let kind = "link" if name == "a" else ("emphasis" if name == "em" or name == "i" else "strong")
                var child = try append(self.doc, parent, kind)
                defer child.close()
                if name == "a":
                    if let href = token.attr("href"):
                        try set_text(self.doc, child.view(), "href", href)
                self.position += 1
                try self.inline_content(child.view(), name, depth + 1)
            elif name == "code":
                self.position += 1
                var code: Buffer
                defer code.close()
                while self.position < self.tokens.values.count and not self.at_close("code"):
                    let part = &self.tokens.values.items[self.position]
                    if part.kind == xml.Kind.text:
                        try code.put(part.text.bytes())
                    self.position += 1
                if self.position < self.tokens.values.count:
                    self.position += 1
                var child = try append(self.doc, parent, "code")
                defer child.close()
                try set_text(self.doc, child.view(), "text", code.view())
            else:
                self.position += 1
                try self.inline_content(parent, name, depth + 1)

    func pre(parent: str) -> !:
        var language: str = ""
        var code: Buffer
        defer code.close()
        while self.position < self.tokens.values.count and not self.at_close("pre"):
            let token = &self.tokens.values.items[self.position]
            if token.kind == xml.Kind.open and token.name.view() == "code":
                if let classes = token.attr("class"):
                    if starts(classes, "language-"):
                        language = (str)classes.bytes[9..]
            elif token.kind == xml.Kind.text:
                try code.put(token.text.bytes())
            self.position += 1
        if self.position < self.tokens.values.count:
            self.position += 1
        if code.used > 0 and code.data[code.used - 1] == '\n':
            code.used -= 1
        var child = try append(self.doc, parent, "codeblock")
        defer child.close()
        try set_text(self.doc, child.view(), "language", language)
        try set_text(self.doc, child.view(), "text", code.view())

    func raw(parent: str) -> !:
        let first = &self.tokens.values.items[self.position]
        let name = first.name.view()
        let start = first.start
        var end = first.end
        var level: usize = 0
        var closed = false
        while self.position < self.tokens.values.count:
            let token = &self.tokens.values.items[self.position]
            self.position += 1
            if token.name.view() != name:
                continue
            if token.kind == xml.Kind.open:
                level += 1
            elif token.kind == xml.Kind.close:
                if level > 0:
                    level -= 1
                if level == 0:
                    end = token.end
                    closed = true
                    break
            elif token.kind == xml.Kind.empty and level == 0:
                end = token.end
                closed = true
                break
        if not closed:
            error(invalid, "unterminated raw HTML island")
        var child = try append(self.doc, parent, "rawHtml")
        defer child.close()
        try set_text(self.doc, child.view(), "text", (str)self.source.bytes[start..<end])

    func list(parent: str, name: str, depth: usize) -> !:
        var child = try append(self.doc, parent, "definitionList" if name == "dl" else "list")
        defer child.close()
        if name != "dl":
            try set_boolean(self.doc, child.view(), "ordered", name == "ol")
        self.position += 1
        while self.position < self.tokens.values.count and not self.at_close(name):
            let token = &self.tokens.values.items[self.position]
            let item_name = token.name.view()
            self.position += 1
            if token.kind != xml.Kind.open:
                continue
            if name != "dl" and item_name == "li":
                var item = try append(self.doc, child.view(), "item")
                defer item.close()
                try self.blocks(item.view(), "li", depth + 1)
            elif name == "dl" and (item_name == "dt" or item_name == "dd"):
                var item = try append(self.doc, child.view(), "term" if item_name == "dt" else "definition")
                defer item.close()
                if item_name == "dt":
                    try self.inline_content(item.view(), "dt", depth + 1)
                else:
                    try self.blocks(item.view(), "dd", depth + 1)
        if self.position < self.tokens.values.count:
            self.position += 1

    func table(parent: str, depth: usize) -> !:
        var child = try append(self.doc, parent, "table")
        defer child.close()
        self.position += 1
        while self.position < self.tokens.values.count and not self.at_close("table"):
            let token = &self.tokens.values.items[self.position]
            self.position += 1
            if token.kind != xml.Kind.open or token.name.view() != "tr":
                continue
            var row = try append(self.doc, child.view(), "tablerow")
            defer row.close()
            var header = false
            while self.position < self.tokens.values.count and not self.at_close("tr"):
                let cell_token = &self.tokens.values.items[self.position]
                let name = cell_token.name.view()
                self.position += 1
                if cell_token.kind == xml.Kind.open and (name == "td" or name == "th"):
                    header = header or name == "th"
                    var cell = try append(self.doc, row.view(), "tablecell")
                    defer cell.close()
                    try self.inline_content(cell.view(), name, depth + 1)
            if self.position < self.tokens.values.count:
                self.position += 1
            if header:
                try set_boolean(self.doc, row.view(), "header", true)
        if self.position < self.tokens.values.count:
            self.position += 1

    func blocks(parent: str, close: str, depth: usize) -> !:
        if depth > 128:
            error(limit_exceeded, "HTML block nesting exceeds 128")
        while self.position < self.tokens.values.count:
            let token = &self.tokens.values.items[self.position]
            let name = token.name.view()
            if token.kind == xml.Kind.close:
                if name == close:
                    self.position += 1
                return
            if token.kind == xml.Kind.text:
                if blank(token.text.view()):
                    self.position += 1
                else:
                    var child = try append(self.doc, parent, "paragraph")
                    defer child.close()
                    try self.inline_content(child.view(), "", depth + 1)
            elif raw_tag(name):
                try self.raw(parent)
            elif token.kind == xml.Kind.empty:
                if name == "hr":
                    var child = try append(self.doc, parent, "thematicBreak")
                    child.close()
                    self.position += 1
                else:
                    var child = try append(self.doc, parent, "paragraph")
                    defer child.close()
                    try self.inline_content(child.view(), "", depth + 1)
            elif name.length == 2 and name.bytes[0] == 'h' and name.bytes[1] >= '1' and name.bytes[1] <= '6':
                var child = try append(self.doc, parent, "heading")
                defer child.close()
                try set_integer(self.doc, child.view(), "level", (i32)(name.bytes[1] - '0'))
                if let id = token.attr("id"):
                    try set_text(self.doc, child.view(), "id", id)
                self.position += 1
                try self.inline_content(child.view(), name, depth + 1)
            elif name == "p" or name == "figcaption":
                var child = try append(self.doc, parent, "paragraph" if name == "p" else "caption")
                defer child.close()
                self.position += 1
                try self.inline_content(child.view(), name, depth + 1)
            elif name == "ul" or name == "ol" or name == "dl":
                try self.list(parent, name, depth)
            elif name == "table":
                try self.table(parent, depth)
            elif name == "figure" or name == "blockquote" or (name == "div" and class_contains(token, "note")):
                var child = try append(self.doc, parent, "note" if name == "div" else name)
                defer child.close()
                self.position += 1
                try self.blocks(child.view(), name, depth + 1)
            elif name == "pre":
                self.position += 1
                try self.pre(parent)
            elif inline_tag(name):
                var child = try append(self.doc, parent, "paragraph")
                defer child.close()
                try self.inline_content(child.view(), "", depth + 1)
            else:
                self.position += 1
                try self.blocks(parent, name, depth + 1)

pub func decode(bytes: const u8[]) -> DocumentData!:
    let source = try str(bytes)
    var tokens = try xml.tokenize(source, true)
    defer tokens.close()
    var doc = DocumentData()
    errdefer doc.close()
    try doc.add("/document", "document")
    var parser = Parser(source = source, tokens = &tokens, doc = &doc)
    try parser.blocks("/document", "", 0)
    return doc