Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-fonts

Source of release 0.7.1

0.7.1 / tools / shaping_unicode.py

#!/usr/bin/env python3
"""Generate src/shaping/unicode_tables.lucb, the Unicode properties the shaper reads.

usage: tools/shaping_unicode.py UCD_DIRECTORY

UCD_DIRECTORY holds four files of one Unicode version (17.0.0 at present):
UnicodeData.txt and emoji/emoji-data.txt (luce-std carries both under
data/unicode/<version>), and Scripts.txt and PropertyValueAliases.txt from
https://www.unicode.org/Public/<version>/ucd/. emoji-data.txt may sit in the directory
itself or in its emoji/ subdirectory.

The shaper (HarfBuzz's default shaper, src/shaping) needs, per scalar:

- whether it is a mark, and which (general category Mn, Mc or Me);
- its script, as an ISO 15924 code (Common, Inherited and Unknown count as none);
- whether it has the Emoji property.

Each property is a step function: entry `start << 8 | value` says that every scalar from
`start` up to the next entry's start has `value`. Canonical combining classes and
normalization come from luce-std's unicode module, of the same Unicode version.
"""
from pathlib import Path
import re
import sys

ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "src/shaping/unicode_tables.lucb"
MARKS = {"Mn": 1, "Mc": 2, "Me": 3}


def ranges_of(path):
    """(first, last, value) of each data line of a UCD property file."""
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.split("#", 1)[0].strip()
        if not line:
            continue
        fields = [field.strip() for field in line.split(";")]
        if ".." in fields[0]:
            first, last = (int(part, 16) for part in fields[0].split(".."))
        else:
            first = last = int(fields[0], 16)
        yield first, last, fields[1]


def version_of(path):
    match = re.search(r"-(\d+\.\d+\.\d+)\.txt", path.read_text(encoding="utf-8").splitlines()[0])
    return match.group(1)


def marks(directory):
    """The mark kind of every scalar from UnicodeData.txt (marks never come in ranges)."""
    values = [0] * 0x110000
    for line in (directory / "UnicodeData.txt").read_text(encoding="utf-8").splitlines():
        fields = line.split(";")
        if fields[2] in MARKS:
            values[int(fields[0], 16)] = MARKS[fields[2]]
    return values


def scripts(directory):
    """The script index of every scalar, and the ISO codes indexed (index 0: none)."""
    codes = {}
    for line in (directory / "PropertyValueAliases.txt").read_text(encoding="utf-8").splitlines():
        fields = [field.strip() for field in line.split("#", 1)[0].split(";")]
        if len(fields) >= 3 and fields[0] == "sc":
            codes[fields[2]] = fields[1]
    names = [""]
    values = [0] * 0x110000
    for first, last, name in ranges_of(directory / "Scripts.txt"):
        code = codes[name]
        if code in ("Zyyy", "Zinh", "Zzzz"):
            continue
        if code not in names:
            names.append(code)
        index = names.index(code)
        for scalar in range(first, last + 1):
            values[scalar] = index
    return values, names


def emoji(directory):
    path = directory / "emoji-data.txt"
    if not path.exists():
        path = directory / "emoji" / "emoji-data.txt"
    values = [0] * 0x110000
    for first, last, name in ranges_of(path):
        if name == "Emoji":
            for scalar in range(first, last + 1):
                values[scalar] = 1
    return values


def steps(values):
    entries = []
    previous = None
    for scalar, value in enumerate(values):
        if value != previous:
            entries.append(scalar << 8 | value)
            previous = value
    return entries


def table(name, entries, what):
    lines = [f"## {what}", f"let {name}: u32[{len(entries)}] = ["]
    for at in range(0, len(entries), 8):
        lines.append("    " + ", ".join(f"0x{value:08x}" for value in entries[at:at + 8]) + ",")
    lines.append("]")
    return lines


def render(version, mark_steps, script_steps, script_names, emoji_steps):
    lines = [
        "#" + "=" * 94,
        "#",
        "#   unicode_tables - The Unicode properties the shaper reads: marks, scripts and Emoji",
        "#",
        "#   DESCRIPTION:",
        f"#       Generated by tools/shaping_unicode.py from the Unicode {version} data files; do not",
        "#       edit. Each table is a step function: entry `start << 8 | value` says that every",
        "#       scalar from `start` to the next entry's start has `value`. Unicode data: Copyright",
        "#       Unicode, Inc., Unicode License v3.",
        "#",
        "#" + "=" * 94,
        "",
        "## The Unicode version of the tables.",
        f'pub let shaping_unicode_version: str = "{version}"',
        "",
        "## The ISO 15924 code of each script index of script_steps (index 0: Common, Inherited",
        "## or Unknown).",
        f"let script_codes: str[{len(script_names)}] = [",
    ]
    for at in range(0, len(script_names), 12):
        lines.append("    " + ", ".join(f'"{name}"' for name in script_names[at:at + 12]) + ",")
    lines.append("]")
    lines.append("")
    lines += table("mark_steps", mark_steps, "General category of marks: 1 Mn, 2 Mc, 3 Me, 0 not a mark.")
    lines.append("")
    lines += table("script_steps", script_steps, "The Script property, as an index of script_codes.")
    lines.append("")
    lines += table("emoji_steps", emoji_steps, "The Emoji property (emoji-data.txt): 1 for an emoji.")
    return "\n".join(lines) + "\n"


def main():
    if len(sys.argv) != 2:
        sys.exit(__doc__)
    directory = Path(sys.argv[1])
    version = version_of(directory / "Scripts.txt")
    script_values, script_names = scripts(directory)
    text = render(version, steps(marks(directory)), steps(script_values), script_names, steps(emoji(directory)))
    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT.write_text(text)
    print(f"wrote {OUTPUT.relative_to(ROOT)}: Unicode {version}, {len(script_names) - 1} scripts")


if __name__ == "__main__":
    main()