Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.0

0.7.0 / tests / programs / unicode_graphemes / run.py

#!/usr/bin/env python3
"""Check extended graphemes against Unicode 17 boundaries and long context runs."""
from pathlib import Path
import struct
import subprocess
import sys
import tempfile

ROOT = Path(__file__).resolve().parents[3]
SOURCE = Path(__file__).resolve().parent
COMPILER = Path(sys.argv[1]).resolve()
corpus = bytearray()
cases = 0


def add(clusters):
    global cases
    encoded = [cluster.encode("utf-8") for cluster in clusters]
    assert all(encoded)
    text = b"".join(encoded)
    corpus.extend(struct.pack("<II", len(text), len(encoded)))
    corpus.extend(text)
    for cluster in encoded:
        corpus.extend(struct.pack("<I", len(cluster)))
    cases += 1


for line in (ROOT / "data/unicode/17.0.0/auxiliary/GraphemeBreakTest.txt").read_text().splitlines():
    body = line.split("#", 1)[0].strip()
    if not body:
        continue
    clusters, current = [], ""
    for token in body.split():
        if token == "รท":
            if current:
                clusters.append(current)
                current = ""
        elif token != "ร—":
            current += chr(int(token, 16))
    assert not current
    add(clusters)
for clusters in ([], ["\0"], ["a", "\0", "b"], ["\r\n", "\r", "x", "\n"],
                 ["a\u0301"], ["\ufeff", "a"], ["\uffff", "\U0010ffff"],
                 ["๐Ÿ‘ฉ\u200d๐Ÿ‘ฉ\u200d๐Ÿ‘ง\u200d๐Ÿ‘ฆ"], ["๐Ÿ‘ฉ\u200d\u0301", "๐Ÿ‘ฉ"],
                 ["๐Ÿ‡ฆ\u0301", "๐Ÿ‡ง๐Ÿ‡จ", "๐Ÿ‡ฉ"], ["\u0600๐Ÿ‡ฆ๐Ÿ‡ง", "๐Ÿ‡จ"],
                 ["\u1100\u1161\u11a8"], ["\u0915\u094d\u200d\u0915"],
                 ["a" + "\u0301" * 16384],
                 ["\u0600" * 16384 + "a"],
                 ["๐Ÿ‘ฉ" + "\u0301" * 8192 + "\u200d๐Ÿ‘ฉ"],
                 ["\u0915\u094d" * 8192 + "\u0915"],
                 ["๐Ÿ‡ฆ๐Ÿ‡ง"] * 8192 + ["๐Ÿ‡จ"]):
    add(clusters)
assert cases > 700


def run(command):
    subprocess.run([sys.executable, ROOT.parent / "luce-base/tools/run_case.py", "--", *command], cwd=ROOT, check=True)


with tempfile.TemporaryDirectory(prefix="base-unicode-graphemes-") as temporary:
    work = Path(temporary)
    reference = work / "graphemes.bin"
    reference.write_bytes(corpus)
    executable = work / "check"
    for flags in [*[["--native", "--opt", str(level)] for level in range(4)],
                  ["--backend=c"], ["--backend=c", "--release"]]:
        run([COMPILER, "build", SOURCE / "main.lucb", *flags, "-o", executable])
        run([executable, reference])
        print(f"PASS {' '.join(flags)}: {cases} boundary cases", flush=True)