dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / tools / gen_ucd / idna.lucb
#==============================================================================================
#
# idna - generated_ucd_idna.lucb: the UTS #46 IDNA mapping table
#
# DESCRIPTION:
# Reads uts46.txt, ICU's gennorm2 rendering of IdnaMappingTable.txt 17.0.0: mapped
# characters are `XXXX>YYYY` lines, ignored ones map to nothing, disallowed ones to
# U+FFFD, and the valid and deviation lines are kept as comments. The generator turns
# it back into ranges with a status and a mapping.
#
#==============================================================================================
# The statuses of the table, as numbered in the generated fragment.
let idna_valid: u8 = 0
let idna_mapped: u8 = 1
let idna_deviation: u8 = 2
let idna_disallowed: u8 = 3
let idna_ignored: u8 = 4
## The status and mapping of every code point.
struct IdnaTable:
var status: u8[]
var mapping_offset: u32[]
var mapping_length: u8[]
var pool: SequencePool
static func create() -> IdnaTable!:
var table = IdnaTable(status = try new u8[code_point_count],
mapping_offset = try new u32[code_point_count],
mapping_length = try new u8[code_point_count], pool = try SequencePool.create())
# Code points the table does not list are disallowed.
for code_point in 0..<code_point_count:
table.status[code_point] = idna_disallowed
_ = try table.pool.intern([0xffffff])
return table
func parse_uts46(table: IdnaTable*, data: const u8[]) -> !:
var lines = Lines.over(data)
while let line = lines.next():
if line.length == 0:
continue
if line.bytes[0] == '#':
# "# 0000..002C valid # 1.1 ..." and "# 00DF deviation >0073 0073 # 1.1 ..."
let content = trim_spaces((str)line.bytes[1..])
let comment_end = strings.find(content, "#") else continue
let body = trim_spaces((str)content.bytes[..<comment_end])
# The range runs into the status when it is long: "1E922..1E94Avalid".
var range_end: usize = 0
while range_end < body.length and is_hex_range((str)body.bytes[range_end..<range_end + 1]):
range_end += 1
let range_text = (str)body.bytes[..<range_end]
if range_text.length < 4:
continue
var words = Fields.over(trim_spaces((str)body.bytes[range_end..]), ' ')
let status_word = words.next() else continue
if status_word == "valid":
try set_idna(table, parse_range(range_text), idna_valid, "")
elif status_word == "deviation":
let mapping = strings.split_once(body, ">") else trap("a deviation without a mapping")
try set_idna(table, parse_range(range_text), idna_deviation, mapping.1)
continue
let content = trim_spaces(strip_comment(line))
if content.length == 0:
continue
let pair = strings.split_once(content, ">") else trap(f"an unexpected uts46.txt line: {line}")
let range = parse_range(pair.0)
let mapping = trim_spaces(pair.1)
if mapping == "FFFD":
try set_idna(table, range, idna_disallowed, "")
elif mapping.length == 0:
try set_idna(table, range, idna_ignored, "")
else:
try set_idna(table, range, idna_mapped, mapping)
func is_hex_range(text: str) -> bool:
for byte in text.bytes:
if not ((byte >= '0' and byte <= '9') or (byte >= 'A' and byte <= 'F') or byte == '.'):
return false
return true
func set_idna(table: IdnaTable*, range: CodePointRange, status: u8, mapping: str) -> !:
var offset: u32 = 0
var length: u8 = 0
if mapping.length > 0:
var buffer: u32[32]
let code_points = parse_code_points(mapping, buffer)
offset = try table.pool.intern(code_points)
length = (u8)code_points.length
for code_point in (usize)range.first..=(usize)range.last:
table.status[code_point] = status
table.mapping_offset[code_point] = offset
table.mapping_length[code_point] = length
func emit_idna(table: const IdnaTable*) -> str!:
var out: collections.Buffer
try put_header(&out, "generated_ucd_idna", "the UTS #46 IDNA mapping table", [
"Generated by tools/gen_ucd from data/ucd/uts46.txt (IdnaMappingTable.txt 17.0.0 as",
"ICU 78.2 ships it); do not edit.",
"",
"ucd_idna_ranges: ascending ranges of code points with one status and mapping, 19 digits",
"each: first and last code point (6 each), status (1: valid, mapped, deviation,",
"disallowed, ignored), mapping offset into ucd_idna_pool (4) and length (2).",
"",
"Unicode data: Copyright Unicode, Inc., Unicode License v3 (data/ucd/LICENSE.txt).",
])
try out.put_text("\nlet ucd_idna_ranges: str = \"")
var start: usize = 0
var range_count: usize = 0
while start < code_point_count:
var end = start
# Consecutive code points share a range when their status and mapping are equal and
# the mapping is empty (a mapping belongs to one code point).
while end + 1 < code_point_count and table.status[end + 1] == table.status[start] and table.mapping_length[start] == 0 and table.mapping_length[end + 1] == 0:
end += 1
try put_hex(&out, (u64)start, 6)
try put_hex(&out, (u64)end, 6)
try put_hex(&out, (u64)table.status[start], 1)
try put_hex(&out, (u64)table.mapping_offset[start], 4)
try put_hex(&out, (u64)table.mapping_length[start], 2)
range_count += 1
start = end + 1
try out.put_text("\"\n\n# The code points of the mappings, 6 digits each.\nlet ucd_idna_pool: str = \"")
for value in table.pool.values.view():
try put_hex(&out, (u64)value, 6)
try out.put_text("\"\n")
_ = range_count
return out.text()