Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / tools / gen_ucd / idna.lucb

#==============================================================================================
#
#   idna - generated_ucd_idna.lucb: the UTS #46 IDNA mapping table
#
#   DESCRIPTION:
#       Reads uts46.txt, ICU's gennorm2 rendering of IdnaMappingTable.txt 17.0.0: mapped
#       characters are `XXXX>YYYY` lines, ignored ones map to nothing, disallowed ones to
#       U+FFFD, and the valid and deviation lines are kept as comments. The generator turns
#       it back into ranges with a status and a mapping.
#
#==============================================================================================

# The statuses of the table, as numbered in the generated fragment.
let idna_valid: u8 = 0
let idna_mapped: u8 = 1
let idna_deviation: u8 = 2
let idna_disallowed: u8 = 3
let idna_ignored: u8 = 4

## The status and mapping of every code point.
struct IdnaTable:
    var status: u8[]
    var mapping_offset: u32[]
    var mapping_length: u8[]
    var pool: SequencePool

    static func create() -> IdnaTable!:
        var table = IdnaTable(status = try new u8[code_point_count],
                              mapping_offset = try new u32[code_point_count],
                              mapping_length = try new u8[code_point_count], pool = try SequencePool.create())
        # Code points the table does not list are disallowed.
        for code_point in 0..<code_point_count:
            table.status[code_point] = idna_disallowed
        _ = try table.pool.intern([0xffffff])
        return table

func parse_uts46(table: IdnaTable*, data: const u8[]) -> !:
    var lines = Lines.over(data)
    while let line = lines.next():
        if line.length == 0:
            continue
        if line.bytes[0] == '#':
            # "# 0000..002C  valid   # 1.1 ..." and "# 00DF  deviation >0073 0073  # 1.1 ..."
            let content = trim_spaces((str)line.bytes[1..])
            let comment_end = strings.find(content, "#") else continue
            let body = trim_spaces((str)content.bytes[..<comment_end])
            # The range runs into the status when it is long: "1E922..1E94Avalid".
            var range_end: usize = 0
            while range_end < body.length and is_hex_range((str)body.bytes[range_end..<range_end + 1]):
                range_end += 1
            let range_text = (str)body.bytes[..<range_end]
            if range_text.length < 4:
                continue
            var words = Fields.over(trim_spaces((str)body.bytes[range_end..]), ' ')
            let status_word = words.next() else continue
            if status_word == "valid":
                try set_idna(table, parse_range(range_text), idna_valid, "")
            elif status_word == "deviation":
                let mapping = strings.split_once(body, ">") else trap("a deviation without a mapping")
                try set_idna(table, parse_range(range_text), idna_deviation, mapping.1)
            continue
        let content = trim_spaces(strip_comment(line))
        if content.length == 0:
            continue
        let pair = strings.split_once(content, ">") else trap(f"an unexpected uts46.txt line: {line}")
        let range = parse_range(pair.0)
        let mapping = trim_spaces(pair.1)
        if mapping == "FFFD":
            try set_idna(table, range, idna_disallowed, "")
        elif mapping.length == 0:
            try set_idna(table, range, idna_ignored, "")
        else:
            try set_idna(table, range, idna_mapped, mapping)

func is_hex_range(text: str) -> bool:
    for byte in text.bytes:
        if not ((byte >= '0' and byte <= '9') or (byte >= 'A' and byte <= 'F') or byte == '.'):
            return false
    return true

func set_idna(table: IdnaTable*, range: CodePointRange, status: u8, mapping: str) -> !:
    var offset: u32 = 0
    var length: u8 = 0
    if mapping.length > 0:
        var buffer: u32[32]
        let code_points = parse_code_points(mapping, buffer)
        offset = try table.pool.intern(code_points)
        length = (u8)code_points.length
    for code_point in (usize)range.first..=(usize)range.last:
        table.status[code_point] = status
        table.mapping_offset[code_point] = offset
        table.mapping_length[code_point] = length

func emit_idna(table: const IdnaTable*) -> str!:
    var out: collections.Buffer
    try put_header(&out, "generated_ucd_idna", "the UTS #46 IDNA mapping table", [
        "Generated by tools/gen_ucd from data/ucd/uts46.txt (IdnaMappingTable.txt 17.0.0 as",
        "ICU 78.2 ships it); do not edit.",
        "",
        "ucd_idna_ranges: ascending ranges of code points with one status and mapping, 19 digits",
        "each: first and last code point (6 each), status (1: valid, mapped, deviation,",
        "disallowed, ignored), mapping offset into ucd_idna_pool (4) and length (2).",
        "",
        "Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
    ])
    try out.put_text("\nlet ucd_idna_ranges: str = \"")
    var start: usize = 0
    var range_count: usize = 0
    while start < code_point_count:
        var end = start
        # Consecutive code points share a range when their status and mapping are equal and
        # the mapping is empty (a mapping belongs to one code point).
        while end + 1 < code_point_count and table.status[end + 1] == table.status[start] and table.mapping_length[start] == 0 and table.mapping_length[end + 1] == 0:
            end += 1
        try put_hex(&out, (u64)start, 6)
        try put_hex(&out, (u64)end, 6)
        try put_hex(&out, (u64)table.status[start], 1)
        try put_hex(&out, (u64)table.mapping_offset[start], 4)
        try put_hex(&out, (u64)table.mapping_length[start], 2)
        range_count += 1
        start = end + 1
    try out.put_text("\"\n\n# The code points of the mappings, 6 digits each.\nlet ucd_idna_pool: str = \"")
    for value in table.pool.values.view():
        try put_hex(&out, (u64)value, 6)
    try out.put_text("\"\n")
    _ = range_count
    return out.text()