Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-xml

Source of release 0.3.0

0.3.0 / src / xml / decode.lucb

#==============================================================================================
#
#   xml/decode - Encodings: byte order marks, UTF-16, Latin-1 and the encoding declaration
#
#   DESCRIPTION:
#       Every entity, the document included, becomes UTF-8 text before it is parsed: its
#       encoding is found from a byte order mark or the encoding its XML or text
#       declaration names (XML 1.0 §4.3.3 and Appendix F), the bytes are decoded, and
#       the result is checked to hold only XML characters, with every CR LF pair and
#       lone CR made a single LF (§2.11). UTF-8 text needing none of that stays a view of
#       the caller's bytes. UTF-16 (with a byte order mark, or labelled UTF-16LE or
#       UTF-16BE), ISO-8859-1 and US-ASCII are decoded here; any other encoding stops the
#       parse with `unsupported_encoding`, naming it, for a caller that can decode it.
#
#==============================================================================================

## The encoding a byte order mark or the first bytes of an entity show.
enum Encoding:
    utf8
    utf16le
    utf16be

## Decode an entity's bytes to normalised UTF-8. `document` for the document itself;
## `system_id` names an external entity in errors.
func decode_entity(reader: Reader*, source: const u8[], document: bool, system_id: str) -> str!:
    var encoding = Encoding.utf8
    var bom: usize = 0
    if source.length >= 3 and source[0] == 0xEF and source[1] == 0xBB and source[2] == 0xBF:
        bom = 3
    elif source.length >= 2 and source[0] == 0xFE and source[1] == 0xFF:
        encoding = Encoding.utf16be
        bom = 2
    elif source.length >= 2 and source[0] == 0xFF and source[1] == 0xFE:
        encoding = Encoding.utf16le
        bom = 2
    if document and reader.options.assume_utf8:
        if encoding != Encoding.utf8:
            bom = 0
        reader.origin = bom
        return try normalize(reader, (str)source[bom..], document, system_id)
    if bom == 0 and source.length >= 4:
        if source[0] == 0 and source[1] == '<' and source[2] == 0 and source[3] == '?':
            encoding = Encoding.utf16be
        elif source[0] == '<' and source[1] == 0 and source[2] == '?' and source[3] == 0:
            encoding = Encoding.utf16le
        elif source[0] == 0 and source[1] == 0:
            try fail_encoding(reader, "UCS-4", system_id)
        elif source[0] == 0x4C and source[1] == 0x6F and source[2] == 0xA7 and source[3] == 0x94:
            try fail_encoding(reader, "EBCDIC", system_id)
    var text: str = ""
    if encoding == Encoding.utf16le or encoding == Encoding.utf16be:
        text = try decode_utf16(reader, source[bom..], encoding == Encoding.utf16le, system_id)
        let label = declared_label(text.bytes)
        let endian = "UTF-16LE" if encoding == Encoding.utf16le else "UTF-16BE"
        if label.length == 0:
            if bom == 0:
                error(malformed, try at_start(reader, text, system_id, "a UTF-16 entity must begin with a byte order mark"))
        elif bom > 0:
            if not (same_label(label, "UTF-16") or same_label(label, endian) or same_label(label, "ISO-10646-UCS-2")):
                error(malformed, try at_start(reader, text, system_id, f"the entity is in UTF-16 but declares the encoding {label}"))
        elif same_label(label, "UTF-16"):
            error(malformed, try at_start(reader, text, system_id, "a UTF-16 entity must begin with a byte order mark"))
        elif not same_label(label, endian):
            error(malformed, try at_start(reader, text, system_id, f"the entity is in UTF-16 but declares the encoding {label}"))
    else:
        let label = declared_label(source[bom..])
        if label.length == 0 or same_label(label, "UTF-8"):
            text = (str)source[bom..]
            if document:
                reader.origin = bom
        elif bom > 0:
            error(malformed, try at_start(reader, (str)source[bom..], system_id, f"the entity begins with a UTF-8 byte order mark but declares the encoding {label}"))
        elif same_label(label, "UTF-16") or same_label(label, "UTF-16LE") or same_label(label, "UTF-16BE"):
            error(malformed, try at_start(reader, (str)source, system_id, f"the entity declares the encoding {label} but is not in UTF-16"))
        elif is_latin1(label):
            text = try decode_latin1(reader, source)
        elif is_ascii(label):
            for (index, byte) in source.indexed():
                if byte >= 0x80:
                    try push_raw(reader, (str)source[..<index], system_id)
                    error(malformed, try reader.complain(f"the byte {byte} is not US-ASCII, the encoding the entity declares"))
            text = (str)source
        else:
            try check_declaration(reader, source, document, system_id)
            try fail_encoding(reader, label, system_id)
    return try normalize(reader, text, document, system_id)

## Read the XML or text declaration at the start of `data` strictly, before an encoding it
## names is reported unsupported, so a malformed declaration is reported as that.
func check_declaration(reader: Reader*, data: const u8[], document: bool, system_id: str) -> !:
    var end: usize = 0
    while end + 1 < data.length and not (data[end] == '?' and data[end + 1] == '>'):
        if data[end] >= 0x80:
            return
        end += 1
    try reader.frames.push(Frame(text = (str)data[..<end + 2 if end + 2 <= data.length else data.length],
                                 external = system_id.length > 0, system_id = system_id))
    try read_xml_declaration(reader, document)
    reader.frames.used -= 1

## Stop with `unsupported_encoding` for `label`.
func fail_encoding(reader: Reader*, label: str, system_id: str) -> !:
    try push_raw(reader, "", system_id)
    error(unsupported_encoding, try reader.report(unsupported_encoding, f"the encoding {label} is not supported", encoding = label))

## Push a frame over `text` (an entity not yet decoded in full) for an error's position.
func push_raw(reader: Reader*, text: str, system_id: str) -> !:
    try reader.frames.push(Frame(text = text, pos = text.length, external = system_id.length > 0, system_id = system_id))

## Record an error at the start of `text` and answer its message.
func at_start(reader: Reader*, text: str, system_id: str, message: fmt) -> str!:
    try reader.frames.push(Frame(text = text, external = system_id.length > 0, system_id = system_id))
    return try reader.complain(message)

# mark: Decoders ==============================================================================

## UTF-16 `data` as UTF-8 in the reader's arena.
func decode_utf16(reader: Reader*, data: const u8[], little: bool, system_id: str) -> str!:
    var built = &reader.scratch
    built.clear()
    var index: usize = 0
    while index + 1 < data.length:
        var code = (u32)data[index + 1] | ((u32)data[index] << 8)
        if little:
            code = (u32)data[index] | ((u32)data[index + 1] << 8)
        index += 2
        if code >= 0xD800 and code <= 0xDBFF and index + 1 < data.length:
            var low = (u32)data[index + 1] | ((u32)data[index] << 8)
            if little:
                low = (u32)data[index] | ((u32)data[index + 1] << 8)
            if low >= 0xDC00 and low <= 0xDFFF:
                index += 2
                try built.add_scalar(0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00))
                continue
        if code >= 0xD800 and code <= 0xDFFF:
            let partial = try reader.arena.store(built.items[..<built.used])
            try push_raw(reader, partial, system_id)
            error(malformed, try reader.complain("an unpaired UTF-16 surrogate"))
        try built.add_scalar(code)
    if index < data.length:
        let partial = try reader.arena.store(built.items[..<built.used])
        try push_raw(reader, partial, system_id)
        error(malformed, try reader.complain("the UTF-16 text ends in half a character"))
    return try reader.arena.store(built.items[..<built.used])

## ISO-8859-1 `data` as UTF-8 in the reader's arena.
func decode_latin1(reader: Reader*, data: const u8[]) -> str!:
    var built = &reader.scratch
    built.clear()
    for byte in data:
        try built.add_scalar((u32)byte)
    return try reader.arena.store(built.items[..<built.used])

## Check that UTF-8 `text` is valid and holds only XML characters, and normalise its line
## ends; the result is `text` itself when nothing changes.
func normalize(reader: Reader*, text: str, document: bool, system_id: str) -> str!:
    let data = text.bytes
    var carriage = false
    var index: usize = 0
    while index < data.length:
        let byte = data[index]
        if byte < 0x80:
            if byte < 0x20 and byte != '\t' and byte != '\n' and byte != '\r':
                try push_raw(reader, (str)data[..<index], system_id)
                error(malformed, try reader.complain(f"the control character #x{byte:x} is not allowed in XML"))
            carriage = carriage or byte == '\r'
            index += 1
            continue
        let size = utf8_length(data, index)
        if size == 0:
            try push_raw(reader, (str)data[..<index], system_id)
            error(malformed, try reader.complain("the text is not valid UTF-8"))
        let (scalar, _) = decode_utf8(data, index)
        if not is_char(scalar):
            try push_raw(reader, (str)data[..<index], system_id)
            error(malformed, try reader.complain(f"the character #x{scalar:x} is not allowed in XML"))
        index += size
    if not carriage:
        return text
    let target = try reader.arena.reserve(data.length)
    var written: usize = 0
    index = 0
    while index < data.length:
        let byte = data[index]
        index += 1
        if byte == '\r':
            target[written] = '\n'
            written += 1
            if index < data.length and data[index] == '\n':
                index += 1
                if document:
                    try reader.collapsed.push(written)
            continue
        target[written] = byte
        written += 1
    return (str)target[..<written]

# mark: Encoding names ========================================================================

## The encoding an XML or text declaration at the start of `data` names, or "".
func declared_label(data: const u8[]) -> str:
    if data.length < 6 or not ((str)data[..<5] == "<?xml") or not is_space(data[5]):
        return ""
    var index: usize = 5
    let end = data.length if data.length < 512 else 512
    while index + 8 < end:
        if data[index] == '?' and data[index + 1] == '>':
            return ""
        if (str)data[index..<index + 8] == "encoding":
            index += 8
            while index < end and is_space(data[index]):
                index += 1
            if index >= end or data[index] != '=':
                return ""
            index += 1
            while index < end and is_space(data[index]):
                index += 1
            if index >= end or (data[index] != '"' and data[index] != '\''):
                return ""
            let quote = data[index]
            let start = index + 1
            index = start
            while index < end and data[index] != quote:
                if data[index] >= 0x80:
                    return ""
                index += 1
            return (str)data[start..<index] if index < end else ""
        index += 1
    return ""

## Whether encoding names `label` and `expected` are the same, ignoring case.
func same_label(label: str, expected: str) -> bool:
    if label.length != expected.length:
        return false
    for index in 0..<label.length:
        if lower(label.bytes[index]) != lower(expected.bytes[index]):
            return false
    return true

func lower(byte: u8) -> u8:
    return byte + 32 if byte >= 'A' and byte <= 'Z' else byte

## Whether `label` names ISO-8859-1.
func is_latin1(label: str) -> bool:
    for known in ["ISO-8859-1", "ISO_8859-1", "ISO_8859-1:1987", "latin1", "l1", "ISO-IR-100", "CP819", "IBM819", "csISOLatin1"]:
        if same_label(label, known):
            return true
    return false

## Whether `label` names US-ASCII.
func is_ascii(label: str) -> bool:
    for known in ["US-ASCII", "ASCII", "ISO646-US", "ANSI_X3.4-1968", "csASCII", "us", "IBM367", "cp367"]:
        if same_label(label, known):
            return true
    return false