Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / tools / gen_ucd / emit_casing.lucb

#==============================================================================================
#
#   emit_casing - generated_ucd_casing.lucb: case mappings, case closure and width mappings
#
#   DESCRIPTION:
#       The data of ICU's ucase.cpp and of the Halfwidth-Fullwidth transliterator, from the
#       case mapping, case folding and decomposition properties of ppucd.txt.
#
#==============================================================================================

# The conditions of Conditional_Case_Mappings (SpecialCasing.txt), numbered for the table.
let case_conditions: str[6] = ["", "Final_Sigma", "After_Soft_Dotted", "More_Above", "Before_Dot", "After_I"]

# The languages of Conditional_Case_Mappings.
let case_languages: str[4] = ["", "lt", "tr", "az"]

func emit_casing(database: const Database*) -> str!:
    var out: collections.Buffer
    try put_header(&out, "generated_ucd_casing", "case mappings, case closure and width mappings", [
        "Generated by tools/gen_ucd from data/ucd/ppucd.txt (Unicode 17.0.0, the data ICU 78.2",
        "is built from); do not edit.",
        "",
        "ucd_case_mappings: every code point with a case mapping, ascending, 50 digits each:",
        "the code point, its simple lowercase, uppercase, titlecase and case folding (6 digits",
        "each), then its full lowercase, uppercase, titlecase and case folding as an offset into",
        "ucd_case_pool (4 digits) and a length (1 digit); length 0 means the simple mapping.",
        "",
        "Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
    ])
    var pool = try SequencePool.create()
    # Pool offset 0 holds nothing; a real mapping never starts there.
    _ = try pool.intern([0xffffffff])
    var buffer: u32[1]

    try out.put_text("\nlet ucd_case_mappings: str = \"")
    var mapping_count: usize = 0
    for code_point: u32 in 0..<(u32)code_point_count:
        var any = false
        for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf, mapping_lc, mapping_uc, mapping_tc, mapping_cf]:
            if database.mapping(code_point, kind) != none:
                any = true
        if not any:
            continue
        mapping_count += 1
        try put_hex(&out, (u64)code_point, 6)
        for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf]:
            try put_hex(&out, (u64)database.simple_mapping(code_point, kind), 6)
        for kind in [mapping_lc, mapping_uc, mapping_tc, mapping_cf]:
            if let mapped = database.mapping(code_point, kind):
                try put_hex(&out, (u64)try pool.intern(mapped), 4)
                try put_hex(&out, (u64)mapped.length, 1)
            else:
                try put_hex(&out, 0, 5)
    try out.put_text("\"\n")

    # Conditional_Case_Mappings=lt More_Above:lc=0069 0307&tc=0049&uc=0049,tr ...
    try out.put_text("\n# Conditional mappings, 23 digits each: the code point (6), language (1: none, lt, tr,\n# az), condition (1: none, Final_Sigma, After_Soft_Dotted, More_Above, Before_Dot,\n# After_I; +8 for Not_), then lowercase, titlecase and uppercase (offset 4, length 1,\n# length 0xf for the code point itself).\nlet ucd_conditional_case_mappings: str = \"")
    for entry in database.conditional[..<database.conditional_count]:
        var alternatives = Fields.over(entry.text, ',')
        while let alternative = alternatives.next():
            let pair = strings.split_once(alternative, ":") else trap("a conditional mapping without a colon")
            var language: u64 = 0
            var condition: u64 = 0
            var words = Fields.over(pair.0, ' ')
            while let word = words.next():
                if let found = index_of(case_languages, word):
                    language = (u64)found
                elif has_prefix(word, "Not_"):
                    condition = 8 + (u64)(index_of(case_conditions, (str)word.bytes[4..]) else trap(f"unknown condition {word}"))
                else:
                    condition = (u64)(index_of(case_conditions, word) else trap(f"unknown condition {word}"))
            try put_hex(&out, (u64)entry.code_point, 6)
            try put_hex(&out, language, 1)
            try put_hex(&out, condition, 1)
            for wanted in ["lc", "tc", "uc"]:
                var found = false
                var assignments = Fields.over(pair.1, '&')
                while let assignment = assignments.next():
                    let parts = strings.split_once(assignment, "=") else trap("a conditional mapping without =")
                    if parts.0 != wanted:
                        continue
                    found = true
                    var code_points: u32[8]
                    let mapped = parse_code_points(parts.1, code_points)
                    try put_hex(&out, (u64)try pool.intern(mapped), 4)
                    try put_hex(&out, (u64)mapped.length, 1)
                if not found:
                    try put_hex(&out, 0, 4)
                    try put_hex(&out, 15, 1)
    try out.put_text("\"\n")

    try out.put_text("\n# Turkic case folding (CaseFolding.txt status T), 11 digits each: the code point (6),\n# the folding (offset 4, length 1).\nlet ucd_turkic_case_folding: str = \"")
    for code_point: u32 in 0..<(u32)code_point_count:
        if let mapped = database.mapping(code_point, mapping_turkic):
            try put_hex(&out, (u64)code_point, 6)
            try put_hex(&out, (u64)try pool.intern(mapped), 4)
            try put_hex(&out, (u64)mapped.length, 1)
    try out.put_text("\"\n")

    try emit_case_closure(database, &out, &pool)
    try emit_width_mappings(database, &out)

    try out.put_text("\n# The code points of the mappings above, 6 digits each.\nlet ucd_case_pool: str = \"")
    for value in pool.values.view():
        try put_hex(&out, (u64)value & 0xffffff, 6)
    try out.put_text("\"\n")
    _ = buffer
    _ = mapping_count
    return out.text()

## ICU's case closure (ucase_addCaseClosure, UnicodeSet::closeOver(USET_CASE_INSENSITIVE)):
## the code points related by simple case mappings and foldings, transitively. The dotted
## and dotless i are special in ICU: I and i form a class of their own, U+0130 and U+0131
## have no code point relatives.
func emit_case_closure(database: const Database*, out: collections.Buffer*, pool: SequencePool*) -> !:
    # Union-find over the simple mappings.
    var parent = try new u32[code_point_count]
    defer free(parent)
    for code_point in 0..<code_point_count:
        parent[code_point] = (u32)code_point
    for code_point: u32 in 0..<(u32)code_point_count:
        if code_point == 0x130 or code_point == 0x131:
            continue
        for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf]:
            let target = database.simple_mapping(code_point, kind)
            if target == 0x130 or target == 0x131:
                continue
            let a = find_root(parent, code_point)
            let b = find_root(parent, target)
            if a != b:
                parent[(usize)(a if a > b else b)] = a if a < b else b
    # The members of each class, as lists threaded through `next` from the root's `head`.
    let none_: u32 = 0xffffffff
    var head = try new u32[code_point_count]
    defer free(head)
    var tail = try new u32[code_point_count]
    defer free(tail)
    var next = try new u32[code_point_count]
    defer free(next)
    var size = try new u32[code_point_count]
    defer free(size)
    for code_point in 0..<code_point_count:
        head[code_point] = none_
        next[code_point] = none_
    for code_point: u32 in 0..<(u32)code_point_count:
        let root = (usize)find_root(parent, code_point)
        if head[root] == none_:
            head[root] = code_point
        else:
            next[(usize)tail[root]] = code_point
        tail[root] = code_point
        size[root] += 1
    var members = try U32List.create()
    try out.put_text("\n# The case closure of each code point that has one, 12 digits each: the code point (6),\n# the other members of its class (offset into ucd_case_pool 4, count 2).\nlet ucd_case_closure: str = \"")
    for code_point: u32 in 0..<(u32)code_point_count:
        let root = (usize)find_root(parent, code_point)
        if size[root] < 2:
            continue
        members.count = 0
        var member = head[root]
        while member != none_:
            if member != code_point:
                try members.append(member)
            member = next[(usize)member]
        try put_hex(out, (u64)code_point, 6)
        try put_hex(out, (u64)try pool.intern(members.view()), 4)
        try put_hex(out, (u64)members.count, 2)
    try out.put_text("\"\n")

func find_root(parent: u32[], code_point: u32) -> u32:
    var current = code_point
    while parent[(usize)current] != current:
        current = parent[(usize)current]
    return current

## The Halfwidth-Fullwidth transliterator: the inverse of each <wide> decomposition, and each
## <narrow> character to its decomposition (runs of those are recomposed at run time).
func emit_width_mappings(database: const Database*, out: collections.Buffer*) -> !:
    # Sorted by source: collect the wide inverses first.
    var sources = try U32List.create()
    var targets = try U32List.create()
    var narrow = try U32List.create()
    for code_point: u32 in 0..<(u32)code_point_count:
        let dt = database.dt[(usize)code_point]
        if dt != 17 and dt != 10:
            continue
        let mapped = database.mapping(code_point, mapping_dm) else continue
        if mapped.length != 1:
            continue
        if dt == 17:
            # ICU's Halfwidth-Fullwidth rules do not map the white parentheses U+2985 and
            # U+2986 to U+FF5F and U+FF60.
            if code_point == 0xff5f or code_point == 0xff60:
                continue
            try sources.append(mapped[0])
            try targets.append(code_point)
            try narrow.append(0)
        else:
            try sources.append(code_point)
            try targets.append(mapped[0])
            try narrow.append(1)
    # Insertion sort by source (a few hundred entries).
    for index in 1..<sources.count:
        var position = index
        while position > 0 and sources.items[position - 1] > sources.items[position]:
            for list in [&sources, &targets, &narrow]:
                let saved = list.items[position]
                list.items[position] = list.items[position - 1]
                list.items[position - 1] = saved
            position -= 1
    try out.put_text("\n# The Halfwidth-Fullwidth mappings, ascending, 13 digits each: the code point (6), its\n# fullwidth form (6), and 1 when it is a <narrow> character recomposed in runs (1).\nlet ucd_fullwidth_mappings: str = \"")
    for index in 0..<sources.count:
        try put_hex(out, (u64)sources.items[index], 6)
        try put_hex(out, (u64)targets.items[index], 6)
        try put_hex(out, (u64)narrow.items[index], 1)
    try out.put_text("\"\n")

func emit_emoji(database: const Database*) -> str!:
    var out: collections.Buffer
    try put_header(&out, "generated_ucd_emoji", "the multi-code-point sequences of the emoji properties", [
        "Generated by tools/gen_ucd from data/ucd/emoji-sequences.txt and",
        "emoji-zwj-sequences.txt (Unicode 17.0.0); do not edit.",
        "",
        "Each sequence: the binary property it belongs to (2 digits, UProperty), the number of",
        "code points (2 digits), then the code points (6 digits each).",
        "",
        "Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
    ])
    try out.put_text("\nlet ucd_emoji_sequences: str = \"")
    let properties = database.emoji_sequence_properties.view()
    let sequences = database.emoji_sequences.view()
    # The order of icu::UnicodeSet::strings(): by property here, then by UTF-16 code units.
    var order = try new usize[properties.length // 2]
    for index in 0..<order.length:
        order[index] = index * 2
    for index in 1..<order.length:
        let value = order[index]
        var position = index
        while position > 0 and emoji_sequence_less(properties, sequences, value, order[position - 1]):
            order[position] = order[position - 1]
            position -= 1
        order[position] = value
    for index in order:
        let property = properties[index]
        let offset = (usize)properties[index + 1]
        let length = (usize)sequences[offset]
        try put_hex(&out, (u64)property, 2)
        try put_hex(&out, (u64)length, 2)
        for code_point in sequences[offset + 1..<offset + 1 + length]:
            try put_hex(&out, (u64)code_point, 6)
    try out.put_text("\"\n")
    return out.text()

## Whether the sequence entry `a` sorts before `b`: by property, then by UTF-16 code units.
func emoji_sequence_less(properties: const u32[], sequences: const u32[], a: usize, b: usize) -> bool:
    if properties[a] != properties[b]:
        return properties[a] < properties[b]
    var units_a: u16[64]
    var units_b: u16[64]
    let length_a = utf16_units(sequences, (usize)properties[a + 1], units_a)
    let length_b = utf16_units(sequences, (usize)properties[b + 1], units_b)
    var index: usize = 0
    while index < length_a and index < length_b:
        if units_a[index] != units_b[index]:
            return units_a[index] < units_b[index]
        index += 1
    return length_a < length_b

## The UTF-16 code units of the sequence at `offset` of `sequences`.
func utf16_units(sequences: const u32[], offset: usize, units: u16[]) -> usize:
    var count: usize = 0
    for code_point in sequences[offset + 1..<offset + 1 + (usize)sequences[offset]]:
        if code_point >= 0x10000:
            units[count] = (u16)(0xd800 + ((code_point - 0x10000) >> 10))
            units[count + 1] = (u16)(0xdc00 + ((code_point - 0x10000) & 0x3ff))
            count += 2
        else:
            units[count] = (u16)code_point
            count += 1
    return count