Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.3

0.7.3 / tools / unicode_tables.py

#!/usr/bin/env python3
"""Generate compact Unicode 17 tables from vendored, checksum-verified UCD data.

ASCII hexadecimal storage keeps the compiler's bootstrap syntax tree small: one
string per table instead of a syntax node for every integer. Each word has exactly
eight characters. Lookup code decodes individual words without allocating.
"""
import argparse
from functools import lru_cache
import hashlib
import json
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DATA = ROOT / "data/unicode/17.0.0"
OUTPUT = ROOT / "src/unicode/tables.lucb"


def records(name):
    for line in (DATA / name).read_text(encoding="utf-8").splitlines():
        body = line.split("#", 1)[0].strip()
        if body:
            yield [field.strip() for field in body.split(";")]


def bounds(field):
    values = field.split("..")
    return int(values[0], 16), int(values[-1], 16)


def property_ranges(name, property_name, value=1):
    return [(first, last, value) for fields in records(name) if fields[1] == property_name
            for first, last in [bounds(fields[0])]]


def merged_ranges(ranges):
    result = []
    for first, last, value in sorted(ranges):
        assert 0 <= first <= last <= 0x10FFFF
        if result:
            assert result[-1][1] < first, "overlapping property ranges"
            if result[-1][1] + 1 == first and result[-1][2] == value:
                result[-1] = (result[-1][0], last, value)
                continue
        result.append((first, last, value))
    return result


def generate():
    manifest = json.loads((DATA / "manifest.json").read_text(encoding="utf-8"))
    for name, metadata in manifest.items():
        assert hashlib.sha256((DATA / name).read_bytes()).hexdigest() == metadata["sha256"], name
    decompositions, combining, upper, lower = {}, {}, {}, {}
    for fields in records("UnicodeData.txt"):
        scalar = int(fields[0], 16)
        if int(fields[3]):
            combining[scalar] = int(fields[3])
        mapping = fields[5].split()
        if mapping:
            compatibility = mapping[0].startswith("<")
            decompositions[scalar] = (compatibility, tuple(int(value, 16) for value in mapping[int(compatibility):]))
        for target, column in ((upper, 12), (lower, 13)):
            if fields[column]:
                target[scalar] = (int(fields[column], 16),)
    for fields in records("SpecialCasing.txt"):
        if fields[4]:
            # The only locale-independent conditional mapping is handled from
            # text context in Base. Locale-specific casing is deliberately separate.
            assert fields[4] == "Final_Sigma" or fields[4].split()[0] in {"tr", "az", "lt"}
            continue
        scalar = int(fields[0], 16)
        lower[scalar] = tuple(int(value, 16) for value in fields[1].split())
        upper[scalar] = tuple(int(value, 16) for value in fields[3].split())
    folding, turkic = {}, {}
    for fields in records("CaseFolding.txt"):
        scalar = int(fields[0], 16)
        mapping = tuple(int(value, 16) for value in fields[2].split())
        if fields[1] in {"C", "F"}:
            folding[scalar] = mapping
        elif fields[1] == "T":
            turkic[scalar] = mapping

    @lru_cache(None)
    def expand(scalar, compatibility):
        if 0xAC00 <= scalar <= 0xD7A3:
            index = scalar - 0xAC00
            return (0x1100 + index // 588, 0x1161 + (index % 588) // 28) + ((0x11A7 + index % 28,) if index % 28 else ())
        mapping = decompositions.get(scalar)
        if mapping is None or (mapping[0] and not compatibility):
            return (scalar,)
        return tuple(part for value in mapping[1] for part in expand(value, compatibility))

    canonical = {scalar: expand(scalar, False) for scalar in decompositions}
    compatible = {scalar: expand(scalar, True) for scalar in decompositions}
    excluded = set()
    for first, last, _ in property_ranges("DerivedNormalizationProps.txt", "Full_Composition_Exclusion"):
        excluded.update(range(first, last + 1))
    composition = []
    for scalar, (compatibility, mapping) in decompositions.items():
        if not compatibility and scalar not in excluded and len(mapping) == 2:
            assert combining.get(mapping[0], 0) == 0
            composition.append((*mapping, scalar))
    composition.sort()
    assert len({row[:2] for row in composition}) == len(composition)

    grapheme_names = "Other CR LF Control Extend ZWJ Regional_Indicator Prepend SpacingMark L V T LV LVT".split()
    graphemes = []
    for fields in records("auxiliary/GraphemeBreakProperty.txt"):
        first, last = bounds(fields[0])
        graphemes.append((first, last, grapheme_names.index(fields[1])))
    indic_names = {"Consonant": 1, "Linker": 2, "Extend": 3}
    indic = []
    for fields in records("DerivedCoreProperties.txt"):
        if fields[1] == "InCB":
            first, last = bounds(fields[0])
            indic.append((first, last, indic_names[fields[2]]))

    pool, pool_offsets = [], {}
    tables = []

    def emit(name, rows):
        rows = list(rows)
        words = [word for row in rows for word in row]
        assert all(0 <= word <= 0xFFFFFFFF for word in words)
        text = "".join(f"{word:08x}" for word in words)
        tables.extend([f"# {len(rows)} records; {len(rows[0]) if rows else 0} words per record.",
                       f'let {name}: str = "{text}"', ""])

    for name, mapping in (("canonical_mappings", canonical), ("compatibility_mappings", compatible),
                          ("upper_mappings", upper), ("lower_mappings", lower),
                          ("fold_mappings", folding), ("turkic_mappings", turkic)):
        rows = []
        for scalar, sequence in sorted(mapping.items()):
            if sequence == (scalar,):
                continue
            assert 1 <= len(sequence) <= 18
            if sequence not in pool_offsets:
                pool_offsets[sequence] = len(pool)
                pool.extend(sequence)
            rows.append((scalar, pool_offsets[sequence], len(sequence)))
        emit(name, rows)
    emit("mapping_scalars", [(scalar,) for scalar in pool])
    emit("composition_mappings", composition)
    emit("combining_ranges", merged_ranges((scalar, scalar, value) for scalar, value in combining.items()))
    emit("cased_ranges", merged_ranges(property_ranges("DerivedCoreProperties.txt", "Cased")))
    emit("case_ignorable_ranges", merged_ranges(property_ranges("DerivedCoreProperties.txt", "Case_Ignorable")))
    emit("whitespace_ranges", merged_ranges(property_ranges("PropList.txt", "White_Space")))
    emit("grapheme_ranges", merged_ranges(graphemes))
    emit("pictographic_ranges", merged_ranges(property_ranges("emoji/emoji-data.txt", "Extended_Pictographic")))
    emit("indic_ranges", merged_ranges(indic))
    header = (
        "#==============================================================================================\n"
        "#\n"
        "#   tables - Generated Unicode 17.0.0 property tables\n"
        "#\n"
        "#   DESCRIPTION:\n"
        "#       Generated by tools/unicode_tables.py; edit the generator, not these tables.\n"
        "#       Unicode 17.0.0 data: Copyright Unicode, Inc. Licensed under Unicode-3.0. The\n"
        "#       full permission notice is in data/unicode/17.0.0/LICENSE.txt.\n"
        "#\n"
        "#==============================================================================================\n\n"
    )
    return header + "\n".join(tables)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true")
    args = parser.parse_args()
    generated = generate()
    if args.check:
        if not OUTPUT.exists() or OUTPUT.read_text(encoding="utf-8") != generated:
            raise SystemExit("Unicode tables differ; run python3 tools/unicode_tables.py")
    else:
        OUTPUT.parent.mkdir(parents=True, exist_ok=True)
        OUTPUT.write_text(generated, encoding="utf-8", newline="\n")
        print(f"wrote {OUTPUT.relative_to(ROOT)} ({len(generated)} bytes)")


if __name__ == "__main__":
    main()