Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / tools / gen_ucd / module.lucb

#==============================================================================================
#
#   gen_ucd - the generator of web_unicode's Unicode Character Database tables
#
#   DESCRIPTION:
#       Ladybird's LibUnicode is a wrapper over ICU; the luce-browser port replaces the ICU
#       pieces LibWeb needs with tables of its own (DESIGN.md §1.1, §5). This program reads
#       the data ICU 78.2 builds its own tables from, copied into data/ucd, and the locale
#       data ICU 78.2 answers with, dumped into data/icu:
#
#           ucd/ppucd.txt                ICU's preparsed UCD, Unicode 17.0.0: every property
#           ucd/emoji-sequences.txt      the emoji sequence properties (Basic_Emoji, RGI_*)
#           ucd/emoji-zwj-sequences.txt  RGI_Emoji_ZWJ_Sequence
#           ucd/uts46.txt                the UTS #46 IDNA mapping table in gennorm2 syntax
#           ucd/nfc.txt                  ICU's canonical combining classes (gennorm2 syntax)
#           icu/available_locales.txt    icu::Locale::getAvailableLocales
#           icu/likely_subtags.txt       icu::Locale::addLikelySubtags of the tags it lists
#           icu/rtl_scripts.txt          uscript_isRightToLeft
#
#       and writes the generated fragments of the web_unicode module:
#
#           generated_ucd_properties.lucb    per-code-point properties and property names
#           generated_ucd_casing.lucb        case mappings, case closure, width mappings
#           generated_ucd_idna.lucb          the IDNA mapping table
#           generated_ucd_emoji.lucb         the multi-code-point emoji sequences
#           generated_icu_locales.lucb       available locales, likely subtags, RTL scripts
#
#       Usage: gen_ucd DATA_DIRECTORY OUTPUT_DIRECTORY, e.g. gen_ucd data
#       src/web_unicode. tests/generated regenerates into build/tests/generated
#       and compares with the committed fragments.
#
#       The numbering of every enumeration is ICU's (uchar.h, uscript.h), because the C++
#       API of LibUnicode exposes ICU's values (Unicode::GeneralCategory, Property, Script).
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause (see LICENSE)
#
#==============================================================================================

import c
from luce_std import collections, files
import memory
import strings

## The number of code points.
let code_point_count: usize = 0x110000

## Generate every table from `arguments[1]` (the data directory) into `arguments[2]`.
pub func main(arguments: c.str[]) -> i32!:
    if arguments.length != 3:
        print("usage: gen_ucd DATA_DIRECTORY OUTPUT_DIRECTORY")
        return 2
    let data_directory = try str(arguments[1])
    let output_directory = try str(arguments[2])

    var database = try Database.create()
    try parse_ppucd(&database, try read_data(data_directory, "ucd/ppucd.txt"))
    try parse_emoji_sequences(&database, try read_data(data_directory, "ucd/emoji-sequences.txt"))
    try parse_emoji_sequences(&database, try read_data(data_directory, "ucd/emoji-zwj-sequences.txt"))
    parse_nfc_combining_classes(&database, try read_data(data_directory, "ucd/nfc.txt"))
    derive_binary_properties(&database)

    try write_output(output_directory, "generated_ucd_properties.lucb", try emit_properties(&database))
    try write_output(output_directory, "generated_ucd_casing.lucb", try emit_casing(&database))
    try write_output(output_directory, "generated_ucd_emoji.lucb", try emit_emoji(&database))

    var idna = try IdnaTable.create()
    try parse_uts46(&idna, try read_data(data_directory, "ucd/uts46.txt"))
    try write_output(output_directory, "generated_ucd_idna.lucb", try emit_idna(&idna))

    let locales = try emit_locales(try read_data(data_directory, "icu/available_locales.txt"), try read_data(data_directory, "icu/likely_subtags.txt"), try read_data(data_directory, "icu/rtl_scripts.txt"))
    try write_output(output_directory, "generated_icu_locales.lucb", locales)
    return 0

## The bytes of the data file `name` of `directory`; the whole run owns them, so they are
## never released.
func read_data(directory: str, name: str) -> const u8[]!:
    var path: collections.Buffer
    try path.put_text(directory)
    try path.put_text("/")
    try path.put_text(name)
    let data = try files.read(try path.terminated())
    return data.value

## Write `text` to the file `name` of `directory`.
func write_output(directory: str, name: str, text: str) -> !:
    var path: collections.Buffer
    try path.put_text(directory)
    try path.put_text("/")
    try path.put_text(name)
    try files.write(try path.terminated(), text.bytes)