dymokomi/luce-std
Source of release 0.7.3
0.7.3 / tests / unicode_normalization / reference.py
#!/usr/bin/env python3
"""Write normalization.bin into the directory given: every relation in Unicode 17's
NormalizationTest plus stress cases from Python's unicodedata, which main.luc hands to
check.lucb."""
from pathlib import Path
import random
import struct
import sys
import unicodedata
ROOT = Path(__file__).resolve().parents[2]
OUTPUT = Path(sys.argv[1])
expected = {}
relations = 0
def add(form, value, result):
global relations
key = form, value.encode("utf-8")
encoded = result.encode("utf-8")
assert key not in expected or expected[key] == encoded
expected[key] = encoded
relations += 1
for line in (ROOT / "data/unicode/17.0.0/NormalizationTest.txt").read_text(encoding="utf-8").splitlines():
body = line.split("#", 1)[0].strip()
if not body or body.startswith("@"):
continue
columns = ["".join(chr(int(value, 16)) for value in field.split()) for field in body.split(";")[:5]]
source, nfc, nfd, nfkc, nfkd = columns
for index, value in enumerate(columns):
add(0, value, nfc if index < 3 else nfkc)
add(1, value, nfd if index < 3 else nfkd)
add(2, value, nfkc)
add(3, value, nfkd)
assert relations > 300000
forms = ("NFC", "NFD", "NFKC", "NFKD")
samples = ["", "\0", "e\u0301\0", "\ufb01", "\ufeffA", "\uffff\U0010ffff",
"A" + "\u0315\u0300\u05ae" * 4096,
"\u0315\u0300\u05ae" * 4096,
"a\u0301" * 2048, "\u1100\u1161\u11a8" * 2048]
randomizer = random.Random(150057)
# Restrict the independent Python campaign to long-established characters, so
# its installed Unicode version need not equal the pinned version being tested.
pool = [chr(value) for value in range(256)] + [chr(value) for value in range(0x300, 0x370)]
pool += list("\u1100\u1161\u11a8\uac00\uac01\ufb01\u212b\u1e9b\u0323")
for _ in range(2000):
samples.append("".join(randomizer.choice(pool) for _ in range(randomizer.randrange(16))))
for value in samples:
for form, name in enumerate(forms):
add(form, value, unicodedata.normalize(name, value))
corpus = bytearray()
for (form, value), result in sorted(expected.items()):
corpus.extend(struct.pack("<III", form, len(value), len(result)))
corpus.extend(value + result)
(OUTPUT / "normalization.bin").write_bytes(corpus)