dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / src / web_unicode / tests_idna.lucb
#==============================================================================================
#
# tests_idna - UTS #46 processing: IdnaTestV2.txt and Tests/LibUnicode/TestIDNA.cpp
#
# DESCRIPTION:
# The toAsciiN column of IdnaTestV2.txt (Unicode 17.0.0) through the port of ICU's
# UTS46::nameToASCII_UTF8, with every check on (UseSTD3ASCIIRules, CheckHyphens,
# CheckBidi, CheckJoiners, VerifyDnsLength) as ICU runs it, and the cases of Ladybird's TestIDNA.cpp
# through the same core (idna_to_ascii itself returns an AK String, which ak's stubs
# cannot build yet).
#
#==============================================================================================
# The options of the conformance run: nontransitional, every check.
let idna_test_options: u32 = uidna_check_bidi | uidna_check_contextj | uidna_use_std3_rules | uidna_nontransitional_to_ascii | uidna_nontransitional_to_unicode
## Append the UTF-8 of an IdnaTestV2 field, decoding \uXXXX and \x{XXXX}.
func unescape_idna_field(field: const u8[], out: ByteBuffer*):
var index: usize = 0
while index < field.length:
if field[index] == '\\' and index + 1 < field.length and (field[index + 1] == 'u' or field[index + 1] == 'x'):
var position = index + 2
let braced = position < field.length and field[position] == '{'
if braced:
position += 1
var value: u32 = 0
var digits: usize = 0
while position < field.length and (not braced or field[position] != '}') and (braced or digits < 4):
let digit = field[position]
value = value * 16 + (u32)(digit - '0' if digit <= '9' else (digit - 'A' + 10 if digit <= 'F' else digit - 'a' + 10))
digits += 1
position += 1
if braced:
position += 1
encode_utf8(value, out)
index = position
else:
out.append(field[index])
index += 1
## The fields of an IdnaTestV2 line.
type IdnaTestField = const u8[]
type IdnaTestFields = IdnaTestField[7]
## The fields of an IdnaTestV2 line, trimmed.
func idna_test_fields(line: const u8[], fields: IdnaTestFields*) -> usize:
var count: usize = 0
var start: usize = 0
for index in 0..=line.length:
if index == line.length or line[index] == ';' or line[index] == '#':
if count < 7:
var first = start
var last = index
while first < last and (line[first] == ' ' or line[first] == '\t'):
first += 1
while last > first and (line[last - 1] == ' ' or line[last - 1] == '\t'):
last -= 1
(*fields)[count] = line[first..<last]
count += 1
start = index + 1
if index < line.length and line[index] == '#':
break
return count
## Whether `name` ends with the root label (a trailing dot) and its other labels are 1 to 63
## bytes long.
func labels_fit_before_root(name: const u8[]) -> bool:
if name.length == 0 or name[name.length - 1] != '.':
return false
var start: usize = 0
for index in 0..<name.length:
if name[index] == '.':
let length = index - start
if length == 0 or length > 63:
return false
start = index + 1
return true
test "IdnaTestV2.txt toAsciiN":
with memory.heap:
let data = read_test_file("tests/data/ucd/IdnaTestV2.txt") else trap("cannot read IdnaTestV2.txt")
var cases: usize = 0
var failures: usize = 0
var offset: usize = 0
while offset < data.length:
var end = offset
while end < data.length and data[end] != '\n':
end += 1
let line = data[offset..<end]
offset = end + 1
if line.length == 0 or line[0] == '#':
continue
var fields: IdnaTestFields
if idna_test_fields(line, &fields) < 7:
continue
var source = ByteBuffer.create(fields[0].length + 1)
if fields[0] != b"\"\"":
unescape_idna_field(fields[0], &source)
# toAsciiN, defaulting to toUnicode, defaulting to the source; its status,
# defaulting to toUnicode's.
var expected_field = fields[3]
if expected_field.length == 0:
expected_field = fields[1]
var expected = ByteBuffer.create(expected_field.length + 1)
if expected_field.length == 0:
expected = source
elif expected_field != b"\"\"":
unescape_idna_field(expected_field, &expected)
var status = fields[4]
if status.length == 0:
status = fields[2]
var expect_success = status.length == 0 or status == b"[]"
# ICU permits the empty root label after a trailing dot, which the file reports
# as A4_2 (see uts46.cpp processUnicode); so does the port.
if status == b"[A4_2]" and labels_fit_before_root(expected.view()):
expect_success = true
cases += 1
var info = IdnaInfo()
let result = uts46_name_to_ascii_utf8(source.view(), idna_test_options, &info)
let success = not info.failed and info.errors == 0
var ok = success == expect_success
if ok and success:
ok = result.view() == expected.view()
if not ok:
failures += 1
if failures <= 10:
print(f"IDNA test failure ({info.errors}): {(str)line}")
print(f" got {(str)result.view()}")
assert(cases > 6000)
assert(failures == 0)
## Unicode::IDNA::to_ascii's processing of `input` with the default ToAsciiOptions (or
## transitional processing), or none where to_ascii fails.
func to_ascii_with_default_options(input: str, transitional: bool = false) -> ak.String?:
var options = ToAsciiOptions()
to_ascii_options_init_fields(&options)
if transitional:
options.transitional_processing = TransitionalProcessing.yes
var view: ak.Utf8View
ak.utf8_view_construct_string_view(&view, view_of(input))
return idna_to_ascii(view, options) catch:
return none
func expect_to_ascii(input: str, expected: str, transitional: bool = false):
let actual = to_ascii_with_default_options(input, transitional) else trap("to_ascii failed")
if not view_equals(ak.string_bytes_as_string_view(&actual), expected):
print(f"to_ascii({input}): expected {expected}, got {ak.string_view_text(ak.string_bytes_as_string_view(&actual))}")
trap("to_ascii")
# Ported from Tests/LibUnicode/TestIDNA.cpp TEST_CASE(to_ascii).
test "to_ascii":
expect_to_ascii("www.аррӏе.com", "www.xn--80ak6aa92e.com")
expect_to_ascii("ö.com", "xn--nda.com")
expect_to_ascii("o\u{308}.com", "xn--nda.com")
expect_to_ascii("\u{1FACD}.ge", "xn--i29h.ge")
# Select cases from IdnaTestV2.txt
expect_to_ascii("Faß.de", "xn--fa-hia.de")
expect_to_ascii("Faß.de", "fass.de", transitional = true)
expect_to_ascii("¡", "xn--7a")
expect_to_ascii("Bücher.de", "xn--bcher-kva.de")
expect_to_ascii("\u{646}\u{627}\u{645}\u{647}\u{627}\u{6CC}", "xn--mgba3gch31f")
expect_to_ascii("A.b.c。D。", "a.b.c.d.")
expect_to_ascii("βόλος", "xn--nxasmm1c")
expect_to_ascii("βόλος", "xn--nxasmq6b", transitional = true)
assert(to_ascii_with_default_options("xn--o-ccb.com") == none)
assert(to_ascii_with_default_options("wh--f.com") == none)
assert(to_ascii_with_default_options("xn--whf-cec.com") == none)
assert(to_ascii_with_default_options("-whf.com") == none)
assert(to_ascii_with_default_options("whf-.com") == none)