Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.4

0.7.4 / tests / unicode_casing / reference.py

#!/usr/bin/env python3
"""Write casing.bin and properties.bin into the directory given: Unicode casing from the
pinned UCD mappings and Python's contextual examples, which main.luc hands to check.lucb."""
from pathlib import Path
import random
import struct
import sys

ROOT = Path(__file__).resolve().parents[2]
DATA = ROOT / "data/unicode/17.0.0"
OUTPUT = Path(sys.argv[1])


def records(name):
    for line in (DATA / name).read_text(encoding="utf-8").splitlines():
        body = line.split("#", 1)[0].strip()
        if body:
            yield [field.strip() for field in body.split(";")]


def text(values):
    return "".join(chr(int(value, 16)) for value in values.split())


upper, lower, folds, turkic, combining = {}, {}, {}, {}, {}
for row in records("UnicodeData.txt"):
    character = chr(int(row[0], 16))
    combining[ord(character)] = int(row[3])
    if row[12]: upper[character] = text(row[12])
    if row[13]: lower[character] = text(row[13])
for row in records("SpecialCasing.txt"):
    if not row[4]:
        character = chr(int(row[0], 16))
        lower[character], upper[character] = text(row[1]), text(row[3])
for row in records("CaseFolding.txt"):
    character = chr(int(row[0], 16))
    if row[1] in ("C", "F"): folds[character] = text(row[2])
    if row[1] == "T": turkic[character] = text(row[2])

corpus = bytearray()


def add(kind, value, expected):
    raw, result = value.encode("utf-8"), expected.encode("utf-8")
    corpus.extend(struct.pack("<III", kind, len(raw), len(result)))
    corpus.extend(raw + result)


characters = set(upper) | set(lower) | set(folds) | set(turkic)
characters.update(chr(value) for value in (0, 0x10FFFF, 0xFFFF, 0x378, 0xFEFF, 0x1F600))
randomizer = random.Random(1700)
for _ in range(5000):
    value = randomizer.randrange(0x110000)
    if not 0xD800 <= value <= 0xDFFF: characters.add(chr(value))
for character in sorted(characters):
    add(0, character, upper.get(character, character))
    add(1, character, lower.get(character, character))
    add(2, character, folds.get(character, character))
    add(3, character, turkic.get(character, folds.get(character, character)))
for value in ("", "Σ", "ΟΣ", "ΟΣΑ", "ΟΣ\u0301", "ΟΣ\u0301Α", "AΣ' B", "AΣ'B",
              "A\u0345Σ", " \u0345Σ", "AΣ\u0345", "AΣ\u0345B", "Iİıi", "ßẞ", "\0AΣ\0",
              "A" + "\u0301" * 2000 + "Σ", "AΣ" + "\u0301" * 2000 + "B"):
    # These long-established mappings independently exercise the context rules
    # in Python's implementation; newly added characters use the pinned UCD above.
    add(0, value, value.upper())
    add(1, value, value.lower())
    add(2, value, value.casefold())
add(3, "Iİıi", "ıiıi")

whitespace = set()
for row in records("PropList.txt"):
    if row[1] == "White_Space":
        ends = row[0].split("..")
        whitespace.update(range(int(ends[0], 16), int(ends[-1], 16) + 1))
scalars = {0, 0x10FFFF, 0x110000, 0xFFFFFFFF}
for value in whitespace | {key for key, value in combining.items() if value}:
    scalars.update((value-1, value, value+1))
scalars.update(ord(character) for character in characters)
properties = bytearray()
for value in sorted(scalars):
    properties.extend(struct.pack("<III", value, int(value in whitespace), combining.get(value, 0)))

(OUTPUT / "casing.bin").write_bytes(corpus)
(OUTPUT / "properties.bin").write_bytes(properties)