Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.1

0.7.1 / tests / unicode_graphemes / reference.py

#!/usr/bin/env python3
"""Write graphemes.bin into the directory given: extended grapheme boundaries from
Unicode 17's GraphemeBreakTest and long context runs, which main.luc hands to check.lucb."""
from pathlib import Path
import struct
import sys

ROOT = Path(__file__).resolve().parents[2]
OUTPUT = Path(sys.argv[1])
corpus = bytearray()
cases = 0


def add(clusters):
    global cases
    encoded = [cluster.encode("utf-8") for cluster in clusters]
    assert all(encoded)
    text = b"".join(encoded)
    corpus.extend(struct.pack("<II", len(text), len(encoded)))
    corpus.extend(text)
    for cluster in encoded:
        corpus.extend(struct.pack("<I", len(cluster)))
    cases += 1


for line in (ROOT / "data/unicode/17.0.0/auxiliary/GraphemeBreakTest.txt").read_text(encoding="utf-8").splitlines():
    body = line.split("#", 1)[0].strip()
    if not body:
        continue
    clusters, current = [], ""
    for token in body.split():
        if token == "รท":
            if current:
                clusters.append(current)
                current = ""
        elif token != "ร—":
            current += chr(int(token, 16))
    assert not current
    add(clusters)
for clusters in ([], ["\0"], ["a", "\0", "b"], ["\r\n", "\r", "x", "\n"],
                 ["a\u0301"], ["\ufeff", "a"], ["\uffff", "\U0010ffff"],
                 ["๐Ÿ‘ฉ\u200d๐Ÿ‘ฉ\u200d๐Ÿ‘ง\u200d๐Ÿ‘ฆ"], ["๐Ÿ‘ฉ\u200d\u0301", "๐Ÿ‘ฉ"],
                 ["๐Ÿ‡ฆ\u0301", "๐Ÿ‡ง๐Ÿ‡จ", "๐Ÿ‡ฉ"], ["\u0600๐Ÿ‡ฆ๐Ÿ‡ง", "๐Ÿ‡จ"],
                 ["\u1100\u1161\u11a8"], ["\u0915\u094d\u200d\u0915"],
                 ["a" + "\u0301" * 16384],
                 ["\u0600" * 16384 + "a"],
                 ["๐Ÿ‘ฉ" + "\u0301" * 8192 + "\u200d๐Ÿ‘ฉ"],
                 ["\u0915\u094d" * 8192 + "\u0915"],
                 ["๐Ÿ‡ฆ๐Ÿ‡ง"] * 8192 + ["๐Ÿ‡จ"]):
    add(clusters)
assert cases > 700

(OUTPUT / "graphemes.bin").write_bytes(corpus)