dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / tools / gen_ucd / emit_casing.lucb
#==============================================================================================
#
# emit_casing - generated_ucd_casing.lucb: case mappings, case closure and width mappings
#
# DESCRIPTION:
# The data of ICU's ucase.cpp and of the Halfwidth-Fullwidth transliterator, from the
# case mapping, case folding and decomposition properties of ppucd.txt.
#
#==============================================================================================
# The conditions of Conditional_Case_Mappings (SpecialCasing.txt), numbered for the table.
let case_conditions: str[6] = ["", "Final_Sigma", "After_Soft_Dotted", "More_Above", "Before_Dot", "After_I"]
# The languages of Conditional_Case_Mappings.
let case_languages: str[4] = ["", "lt", "tr", "az"]
func emit_casing(database: const Database*) -> str!:
var out: collections.Buffer
try put_header(&out, "generated_ucd_casing", "case mappings, case closure and width mappings", [
"Generated by tools/gen_ucd from data/ucd/ppucd.txt (Unicode 17.0.0, the data ICU 78.2",
"is built from); do not edit.",
"",
"ucd_case_mappings: every code point with a case mapping, ascending, 50 digits each:",
"the code point, its simple lowercase, uppercase, titlecase and case folding (6 digits",
"each), then its full lowercase, uppercase, titlecase and case folding as an offset into",
"ucd_case_pool (4 digits) and a length (1 digit); length 0 means the simple mapping.",
"",
"Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
])
var pool = try SequencePool.create()
# Pool offset 0 holds nothing; a real mapping never starts there.
_ = try pool.intern([0xffffffff])
var buffer: u32[1]
try out.put_text("\nlet ucd_case_mappings: str = \"")
var mapping_count: usize = 0
for code_point: u32 in 0..<(u32)code_point_count:
var any = false
for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf, mapping_lc, mapping_uc, mapping_tc, mapping_cf]:
if database.mapping(code_point, kind) != none:
any = true
if not any:
continue
mapping_count += 1
try put_hex(&out, (u64)code_point, 6)
for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf]:
try put_hex(&out, (u64)database.simple_mapping(code_point, kind), 6)
for kind in [mapping_lc, mapping_uc, mapping_tc, mapping_cf]:
if let mapped = database.mapping(code_point, kind):
try put_hex(&out, (u64)try pool.intern(mapped), 4)
try put_hex(&out, (u64)mapped.length, 1)
else:
try put_hex(&out, 0, 5)
try out.put_text("\"\n")
# Conditional_Case_Mappings=lt More_Above:lc=0069 0307&tc=0049&uc=0049,tr ...
try out.put_text("\n# Conditional mappings, 23 digits each: the code point (6), language (1: none, lt, tr,\n# az), condition (1: none, Final_Sigma, After_Soft_Dotted, More_Above, Before_Dot,\n# After_I; +8 for Not_), then lowercase, titlecase and uppercase (offset 4, length 1,\n# length 0xf for the code point itself).\nlet ucd_conditional_case_mappings: str = \"")
for entry in database.conditional[..<database.conditional_count]:
var alternatives = Fields.over(entry.text, ',')
while let alternative = alternatives.next():
let pair = strings.split_once(alternative, ":") else trap("a conditional mapping without a colon")
var language: u64 = 0
var condition: u64 = 0
var words = Fields.over(pair.0, ' ')
while let word = words.next():
if let found = index_of(case_languages, word):
language = (u64)found
elif has_prefix(word, "Not_"):
condition = 8 + (u64)(index_of(case_conditions, (str)word.bytes[4..]) else trap(f"unknown condition {word}"))
else:
condition = (u64)(index_of(case_conditions, word) else trap(f"unknown condition {word}"))
try put_hex(&out, (u64)entry.code_point, 6)
try put_hex(&out, language, 1)
try put_hex(&out, condition, 1)
for wanted in ["lc", "tc", "uc"]:
var found = false
var assignments = Fields.over(pair.1, '&')
while let assignment = assignments.next():
let parts = strings.split_once(assignment, "=") else trap("a conditional mapping without =")
if parts.0 != wanted:
continue
found = true
var code_points: u32[8]
let mapped = parse_code_points(parts.1, code_points)
try put_hex(&out, (u64)try pool.intern(mapped), 4)
try put_hex(&out, (u64)mapped.length, 1)
if not found:
try put_hex(&out, 0, 4)
try put_hex(&out, 15, 1)
try out.put_text("\"\n")
try out.put_text("\n# Turkic case folding (CaseFolding.txt status T), 11 digits each: the code point (6),\n# the folding (offset 4, length 1).\nlet ucd_turkic_case_folding: str = \"")
for code_point: u32 in 0..<(u32)code_point_count:
if let mapped = database.mapping(code_point, mapping_turkic):
try put_hex(&out, (u64)code_point, 6)
try put_hex(&out, (u64)try pool.intern(mapped), 4)
try put_hex(&out, (u64)mapped.length, 1)
try out.put_text("\"\n")
try emit_case_closure(database, &out, &pool)
try emit_width_mappings(database, &out)
try out.put_text("\n# The code points of the mappings above, 6 digits each.\nlet ucd_case_pool: str = \"")
for value in pool.values.view():
try put_hex(&out, (u64)value & 0xffffff, 6)
try out.put_text("\"\n")
_ = buffer
_ = mapping_count
return out.text()
## ICU's case closure (ucase_addCaseClosure, UnicodeSet::closeOver(USET_CASE_INSENSITIVE)):
## the code points related by simple case mappings and foldings, transitively. The dotted
## and dotless i are special in ICU: I and i form a class of their own, U+0130 and U+0131
## have no code point relatives.
func emit_case_closure(database: const Database*, out: collections.Buffer*, pool: SequencePool*) -> !:
# Union-find over the simple mappings.
var parent = try new u32[code_point_count]
defer free(parent)
for code_point in 0..<code_point_count:
parent[code_point] = (u32)code_point
for code_point: u32 in 0..<(u32)code_point_count:
if code_point == 0x130 or code_point == 0x131:
continue
for kind in [mapping_slc, mapping_suc, mapping_stc, mapping_scf]:
let target = database.simple_mapping(code_point, kind)
if target == 0x130 or target == 0x131:
continue
let a = find_root(parent, code_point)
let b = find_root(parent, target)
if a != b:
parent[(usize)(a if a > b else b)] = a if a < b else b
# The members of each class, as lists threaded through `next` from the root's `head`.
let none_: u32 = 0xffffffff
var head = try new u32[code_point_count]
defer free(head)
var tail = try new u32[code_point_count]
defer free(tail)
var next = try new u32[code_point_count]
defer free(next)
var size = try new u32[code_point_count]
defer free(size)
for code_point in 0..<code_point_count:
head[code_point] = none_
next[code_point] = none_
for code_point: u32 in 0..<(u32)code_point_count:
let root = (usize)find_root(parent, code_point)
if head[root] == none_:
head[root] = code_point
else:
next[(usize)tail[root]] = code_point
tail[root] = code_point
size[root] += 1
var members = try U32List.create()
try out.put_text("\n# The case closure of each code point that has one, 12 digits each: the code point (6),\n# the other members of its class (offset into ucd_case_pool 4, count 2).\nlet ucd_case_closure: str = \"")
for code_point: u32 in 0..<(u32)code_point_count:
let root = (usize)find_root(parent, code_point)
if size[root] < 2:
continue
members.count = 0
var member = head[root]
while member != none_:
if member != code_point:
try members.append(member)
member = next[(usize)member]
try put_hex(out, (u64)code_point, 6)
try put_hex(out, (u64)try pool.intern(members.view()), 4)
try put_hex(out, (u64)members.count, 2)
try out.put_text("\"\n")
func find_root(parent: u32[], code_point: u32) -> u32:
var current = code_point
while parent[(usize)current] != current:
current = parent[(usize)current]
return current
## The Halfwidth-Fullwidth transliterator: the inverse of each <wide> decomposition, and each
## <narrow> character to its decomposition (runs of those are recomposed at run time).
func emit_width_mappings(database: const Database*, out: collections.Buffer*) -> !:
# Sorted by source: collect the wide inverses first.
var sources = try U32List.create()
var targets = try U32List.create()
var narrow = try U32List.create()
for code_point: u32 in 0..<(u32)code_point_count:
let dt = database.dt[(usize)code_point]
if dt != 17 and dt != 10:
continue
let mapped = database.mapping(code_point, mapping_dm) else continue
if mapped.length != 1:
continue
if dt == 17:
# ICU's Halfwidth-Fullwidth rules do not map the white parentheses U+2985 and
# U+2986 to U+FF5F and U+FF60.
if code_point == 0xff5f or code_point == 0xff60:
continue
try sources.append(mapped[0])
try targets.append(code_point)
try narrow.append(0)
else:
try sources.append(code_point)
try targets.append(mapped[0])
try narrow.append(1)
# Insertion sort by source (a few hundred entries).
for index in 1..<sources.count:
var position = index
while position > 0 and sources.items[position - 1] > sources.items[position]:
for list in [&sources, &targets, &narrow]:
let saved = list.items[position]
list.items[position] = list.items[position - 1]
list.items[position - 1] = saved
position -= 1
try out.put_text("\n# The Halfwidth-Fullwidth mappings, ascending, 13 digits each: the code point (6), its\n# fullwidth form (6), and 1 when it is a <narrow> character recomposed in runs (1).\nlet ucd_fullwidth_mappings: str = \"")
for index in 0..<sources.count:
try put_hex(out, (u64)sources.items[index], 6)
try put_hex(out, (u64)targets.items[index], 6)
try put_hex(out, (u64)narrow.items[index], 1)
try out.put_text("\"\n")
func emit_emoji(database: const Database*) -> str!:
var out: collections.Buffer
try put_header(&out, "generated_ucd_emoji", "the multi-code-point sequences of the emoji properties", [
"Generated by tools/gen_ucd from data/ucd/emoji-sequences.txt and",
"emoji-zwj-sequences.txt (Unicode 17.0.0); do not edit.",
"",
"Each sequence: the binary property it belongs to (2 digits, UProperty), the number of",
"code points (2 digits), then the code points (6 digits each).",
"",
"Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
])
try out.put_text("\nlet ucd_emoji_sequences: str = \"")
let properties = database.emoji_sequence_properties.view()
let sequences = database.emoji_sequences.view()
# The order of icu::UnicodeSet::strings(): by property here, then by UTF-16 code units.
var order = try new usize[properties.length // 2]
for index in 0..<order.length:
order[index] = index * 2
for index in 1..<order.length:
let value = order[index]
var position = index
while position > 0 and emoji_sequence_less(properties, sequences, value, order[position - 1]):
order[position] = order[position - 1]
position -= 1
order[position] = value
for index in order:
let property = properties[index]
let offset = (usize)properties[index + 1]
let length = (usize)sequences[offset]
try put_hex(&out, (u64)property, 2)
try put_hex(&out, (u64)length, 2)
for code_point in sequences[offset + 1..<offset + 1 + length]:
try put_hex(&out, (u64)code_point, 6)
try out.put_text("\"\n")
return out.text()
## Whether the sequence entry `a` sorts before `b`: by property, then by UTF-16 code units.
func emoji_sequence_less(properties: const u32[], sequences: const u32[], a: usize, b: usize) -> bool:
if properties[a] != properties[b]:
return properties[a] < properties[b]
var units_a: u16[64]
var units_b: u16[64]
let length_a = utf16_units(sequences, (usize)properties[a + 1], units_a)
let length_b = utf16_units(sequences, (usize)properties[b + 1], units_b)
var index: usize = 0
while index < length_a and index < length_b:
if units_a[index] != units_b[index]:
return units_a[index] < units_b[index]
index += 1
return length_a < length_b
## The UTF-16 code units of the sequence at `offset` of `sequences`.
func utf16_units(sequences: const u32[], offset: usize, units: u16[]) -> usize:
var count: usize = 0
for code_point in sequences[offset + 1..<offset + 1 + (usize)sequences[offset]]:
if code_point >= 0x10000:
units[count] = (u16)(0xd800 + ((code_point - 0x10000) >> 10))
units[count + 1] = (u16)(0xdc00 + ((code_point - 0x10000) & 0x3ff))
count += 2
else:
units[count] = (u16)code_point
count += 1
return count