dymokomi/luce-fonts
Source of release 0.7.1
0.7.1 / tools / shaping_unicode.py
#!/usr/bin/env python3
"""Generate src/shaping/unicode_tables.lucb, the Unicode properties the shaper reads.
usage: tools/shaping_unicode.py UCD_DIRECTORY
UCD_DIRECTORY holds four files of one Unicode version (17.0.0 at present):
UnicodeData.txt and emoji/emoji-data.txt (luce-std carries both under
data/unicode/<version>), and Scripts.txt and PropertyValueAliases.txt from
https://www.unicode.org/Public/<version>/ucd/. emoji-data.txt may sit in the directory
itself or in its emoji/ subdirectory.
The shaper (HarfBuzz's default shaper, src/shaping) needs, per scalar:
- whether it is a mark, and which (general category Mn, Mc or Me);
- its script, as an ISO 15924 code (Common, Inherited and Unknown count as none);
- whether it has the Emoji property.
Each property is a step function: entry `start << 8 | value` says that every scalar from
`start` up to the next entry's start has `value`. Canonical combining classes and
normalization come from luce-std's unicode module, of the same Unicode version.
"""
from pathlib import Path
import re
import sys
ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "src/shaping/unicode_tables.lucb"
MARKS = {"Mn": 1, "Mc": 2, "Me": 3}
def ranges_of(path):
"""(first, last, value) of each data line of a UCD property file."""
for line in path.read_text(encoding="utf-8").splitlines():
line = line.split("#", 1)[0].strip()
if not line:
continue
fields = [field.strip() for field in line.split(";")]
if ".." in fields[0]:
first, last = (int(part, 16) for part in fields[0].split(".."))
else:
first = last = int(fields[0], 16)
yield first, last, fields[1]
def version_of(path):
match = re.search(r"-(\d+\.\d+\.\d+)\.txt", path.read_text(encoding="utf-8").splitlines()[0])
return match.group(1)
def marks(directory):
"""The mark kind of every scalar from UnicodeData.txt (marks never come in ranges)."""
values = [0] * 0x110000
for line in (directory / "UnicodeData.txt").read_text(encoding="utf-8").splitlines():
fields = line.split(";")
if fields[2] in MARKS:
values[int(fields[0], 16)] = MARKS[fields[2]]
return values
def scripts(directory):
"""The script index of every scalar, and the ISO codes indexed (index 0: none)."""
codes = {}
for line in (directory / "PropertyValueAliases.txt").read_text(encoding="utf-8").splitlines():
fields = [field.strip() for field in line.split("#", 1)[0].split(";")]
if len(fields) >= 3 and fields[0] == "sc":
codes[fields[2]] = fields[1]
names = [""]
values = [0] * 0x110000
for first, last, name in ranges_of(directory / "Scripts.txt"):
code = codes[name]
if code in ("Zyyy", "Zinh", "Zzzz"):
continue
if code not in names:
names.append(code)
index = names.index(code)
for scalar in range(first, last + 1):
values[scalar] = index
return values, names
def emoji(directory):
path = directory / "emoji-data.txt"
if not path.exists():
path = directory / "emoji" / "emoji-data.txt"
values = [0] * 0x110000
for first, last, name in ranges_of(path):
if name == "Emoji":
for scalar in range(first, last + 1):
values[scalar] = 1
return values
def steps(values):
entries = []
previous = None
for scalar, value in enumerate(values):
if value != previous:
entries.append(scalar << 8 | value)
previous = value
return entries
def table(name, entries, what):
lines = [f"## {what}", f"let {name}: u32[{len(entries)}] = ["]
for at in range(0, len(entries), 8):
lines.append(" " + ", ".join(f"0x{value:08x}" for value in entries[at:at + 8]) + ",")
lines.append("]")
return lines
def render(version, mark_steps, script_steps, script_names, emoji_steps):
lines = [
"#" + "=" * 94,
"#",
"# unicode_tables - The Unicode properties the shaper reads: marks, scripts and Emoji",
"#",
"# DESCRIPTION:",
f"# Generated by tools/shaping_unicode.py from the Unicode {version} data files; do not",
"# edit. Each table is a step function: entry `start << 8 | value` says that every",
"# scalar from `start` to the next entry's start has `value`. Unicode data: Copyright",
"# Unicode, Inc., Unicode License v3.",
"#",
"#" + "=" * 94,
"",
"## The Unicode version of the tables.",
f'pub let shaping_unicode_version: str = "{version}"',
"",
"## The ISO 15924 code of each script index of script_steps (index 0: Common, Inherited",
"## or Unknown).",
f"let script_codes: str[{len(script_names)}] = [",
]
for at in range(0, len(script_names), 12):
lines.append(" " + ", ".join(f'"{name}"' for name in script_names[at:at + 12]) + ",")
lines.append("]")
lines.append("")
lines += table("mark_steps", mark_steps, "General category of marks: 1 Mn, 2 Mc, 3 Me, 0 not a mark.")
lines.append("")
lines += table("script_steps", script_steps, "The Script property, as an index of script_codes.")
lines.append("")
lines += table("emoji_steps", emoji_steps, "The Emoji property (emoji-data.txt): 1 for an emoji.")
return "\n".join(lines) + "\n"
def main():
if len(sys.argv) != 2:
sys.exit(__doc__)
directory = Path(sys.argv[1])
version = version_of(directory / "Scripts.txt")
script_values, script_names = scripts(directory)
text = render(version, steps(marks(directory)), steps(script_values), script_names, steps(emoji(directory)))
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
OUTPUT.write_text(text)
print(f"wrote {OUTPUT.relative_to(ROOT)}: Unicode {version}, {len(script_names) - 1} scripts")
if __name__ == "__main__":
main()