dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / web_unicode / idna.lucb
#==============================================================================================
#
# idna - Unicode::IDNA::to_ascii: UTS #46 processing of domain names
#
# DESCRIPTION:
# Port of LibUnicode/IDNA.cpp (lines 1-65), and of the ICU code it calls, which the
# port replaces: ICU 78's uts46.cpp (UTS46::nameToASCII_UTF8 and its helpers) and
# punycode.cpp (u_strToPunycode, u_strFromPunycode), © Unicode, Inc. and others,
# Unicode License v3. The uts46 normalizer (uts46.nrm: the UTS #46 mapping followed by
# NFC) is generated_ucd_idna.lucb's table and luce-std's NFC. ICU works in UTF-16; the
# port works in code points, which only changes the unit of internal offsets.
#
#==============================================================================================
# mark: Unicode::IDNA ===========================================================================
# The options of ICU's uidna.h.
let uidna_check_bidi: u32 = 4
let uidna_check_contextj: u32 = 8
let uidna_nontransitional_to_ascii: u32 = 0x10
let uidna_nontransitional_to_unicode: u32 = 0x20
let uidna_use_std3_rules: u32 = 2
# The errors of ICU's uidna.h.
let uidna_error_empty_label: u32 = 1
let uidna_error_label_too_long: u32 = 2
let uidna_error_domain_name_too_long: u32 = 4
let uidna_error_leading_hyphen: u32 = 8
let uidna_error_trailing_hyphen: u32 = 0x10
let uidna_error_hyphen_3_4: u32 = 0x20
let uidna_error_leading_combining_mark: u32 = 0x40
let uidna_error_disallowed: u32 = 0x80
let uidna_error_punycode: u32 = 0x100
let uidna_error_label_has_dot: u32 = 0x200
let uidna_error_invalid_ace_label: u32 = 0x400
let uidna_error_bidi: u32 = 0x800
let uidna_error_contextj: u32 = 0x1000
## Ported from Libraries/LibUnicode/IDNA.cpp:16 Unicode::IDNA::to_ascii.
# https://www.unicode.org/reports/tr46/#ToASCII
pub func idna_to_ascii(domain_name: ak.Utf8View, options: ToAsciiOptions) -> ak.String!:
var icu_options: u32 = 0
if options.check_bidi == CheckBidi.yes:
icu_options |= uidna_check_bidi
if options.check_joiners == CheckJoiners.yes:
icu_options |= uidna_check_contextj
if options.use_std3_ascii_rules == UseStd3AsciiRules.yes:
icu_options |= uidna_use_std3_rules
if options.transitional_processing == TransitionalProcessing.no:
icu_options |= uidna_nontransitional_to_ascii | uidna_nontransitional_to_unicode
# icu::IDNA::createUTS46Instance(icu_options) and nameToASCII_UTF8.
var info = IdnaInfo()
let result = uts46_name_to_ascii_utf8(view_bytes(domain_name.m_string), icu_options, &info)
var errors = info.errors
if options.check_hyphens == CheckHyphens.no:
errors &= ~uidna_error_hyphen_3_4
errors &= ~uidna_error_leading_hyphen
errors &= ~uidna_error_trailing_hyphen
if options.verify_dns_length == VerifyDnsLength.no:
errors &= ~uidna_error_empty_label
errors &= ~uidna_error_label_too_long
errors &= ~uidna_error_domain_name_too_long
if options.ignore_invalid_punycode == IgnoreInvalidPunycode.yes:
errors &= ~uidna_error_punycode
if info.failed or errors != 0:
error(ak.ak_error, "Unable to convert domain to ASCII")
return ak.string_from_utf8(ak.StringView(m_characters = result.items.data, m_length = result.count)) else trap("MUST: StringBuilder::to_string")
# mark: UTS46 (ICU uts46.cpp) ===================================================================
## icu::IDNAInfo, and ICU's UErrorCode failure of the whole operation (`failed`).
struct IdnaInfo:
var errors: u32
var label_errors: u32
var is_trans_diff: bool
var is_bidi: bool
var is_ok_bidi: bool = true
var failed: bool
# Severe errors which usually result in a U+FFFD replacement character in the result string.
let severe_errors: u32 = uidna_error_leading_combining_mark | uidna_error_disallowed | uidna_error_punycode | uidna_error_label_has_dot | uidna_error_invalid_ace_label
# UTS #46 data for ASCII characters.
# The normalizer (using uts46.nrm) maps uppercase ASCII letters to lowercase
# and passes through all other ASCII characters.
# If UIDNA_USE_STD3_RULES is set, then non-LDH characters are disallowed
# using this data.
# The ASCII fastpath also uses this data.
# Values: -1=disallowed 0==valid 1==mapped (lowercase)
func ascii_data(c: u32) -> i32:
if c == 0x2d or c == 0x2e or (c >= 0x30 and c <= 0x39) or (c >= 0x61 and c <= 0x7a):
return 0
if c >= 0x41 and c <= 0x5a:
return 1
return -1
## The uts46 Normalizer2: the UTS #46 mapping, then NFC.
func uts46_normalize(source: const u32[]) -> CodePointBuffer:
var mapped = CodePointBuffer.create(source.length + 1)
for code_point in source:
let entry = idna_range(code_point)
let status = ucd_hex(ucd_idna_ranges, entry + 12, 1)
if status == idna_status_mapped:
let offset = (usize)ucd_hex(ucd_idna_ranges, entry + 13, 4)
let length = (usize)ucd_hex(ucd_idna_ranges, entry + 17, 2)
for index in 0..<length:
mapped.append(ucd_hex(ucd_idna_pool, (offset + index) * 6, 6))
elif status == idna_status_disallowed:
mapped.append(0xfffd)
elif status != idna_status_ignored:
# valid, and deviation characters (handled by mapDevChars)
mapped.append(code_point)
return normalize_code_points(mapped.view(), NormalizationForm.nfc)
# The statuses of ucd_idna_ranges.
let idna_status_valid: u32 = 0
let idna_status_mapped: u32 = 1
let idna_status_deviation: u32 = 2
let idna_status_disallowed: u32 = 3
let idna_status_ignored: u32 = 4
## The digit offset of the ucd_idna_ranges entry containing `code_point`.
func idna_range(code_point: u32) -> usize:
var low: usize = 0
var high = ucd_idna_ranges.length // 19
while low + 1 < high:
let middle = low + (high - low) // 2
if ucd_hex(ucd_idna_ranges, middle * 19, 6) <= code_point:
low = middle
else:
high = middle
return low * 19
## Whether uts46_normalize leaves `text` unchanged (Normalizer2::isNormalized).
func uts46_is_normalized(text: const u32[]) -> bool:
let normalized = uts46_normalize(text)
if normalized.count != text.length:
return false
for index in 0..<text.length:
if normalized.items[index] != text[index]:
return false
return true
## UTS46::processUTF8 with isLabel=false and toASCII=true (nameToASCII_UTF8): the UTF-8 result.
func uts46_name_to_ascii_utf8(source: const u8[], options: u32, info: IdnaInfo*) -> ByteBuffer:
let is_label = false
let to_ascii = true
var dest = ByteBuffer.create(source.length + 20)
let src_length = source.length
if src_length == 0:
info.errors |= uidna_error_empty_label
return dest
var dest_string = CodePointBuffer.create(src_length + 1)
var label_start: usize = 0
if src_length <= 256:
# ASCII fastpath
var dest_array = ByteBuffer.create(src_length + 1)
let disallow_non_ldh_dot = (options & uidna_use_std3_rules) != 0
var i: usize = 0
while true:
if i == src_length:
if to_ascii:
if i - label_start > 63:
info.label_errors |= uidna_error_label_too_long
# There is a trailing dot if labelStart==i.
if not is_label and i >= 254 and (i > 254 or label_start < i):
info.errors |= uidna_error_domain_name_too_long
info.errors |= info.label_errors
return dest_array
let c = (u32)source[i]
if c > 0x7f:
break
let c_data = ascii_data(c)
if c_data > 0:
# Lowercase an uppercase ASCII letter.
dest_array.append((u8)(c + 0x20))
elif c_data < 0 and disallow_non_ldh_dot:
# Replacing with U+FFFD can be complicated for toASCII.
break
else:
dest_array.append((u8)c)
if c == 0x2d:
# hyphen
if i == label_start + 3 and source[i - 1] == 0x2d:
# "??--..." is Punycode or forbidden.
break
if i == label_start:
# label starts with "-"
info.label_errors |= uidna_error_leading_hyphen
if i + 1 == src_length or source[i + 1] == 0x2e:
# label ends with "-"
info.label_errors |= uidna_error_trailing_hyphen
elif c == 0x2e:
# dot
if is_label:
# Replacing with U+FFFD can be complicated for toASCII.
break
if i == label_start:
info.label_errors |= uidna_error_empty_label
if to_ascii and i - label_start > 63:
info.label_errors |= uidna_error_label_too_long
info.errors |= info.label_errors
info.label_errors = 0
label_start = i + 1
i += 1
info.errors |= info.label_errors
# Convert the processed ASCII prefix of the current label to UTF-16 (code points).
let mapping_start = dest_array.count - label_start
for byte in dest_array.view()[label_start..]:
dest_string.append((u32)byte)
# Output the previous ASCII labels and process the rest of src in UTF-16.
for byte in dest_array.view()[..<label_start]:
dest.append(byte)
var rest = CodePointBuffer.create(src_length - label_start + 1)
decode_utf8(source[label_start..], &rest, none)
uts46_process_unicode(rest.view(), 0, mapping_start, is_label, to_ascii, &dest_string, options, info)
else:
# src is too long for the ASCII fastpath implementation.
var all = CodePointBuffer.create(src_length + 1)
decode_utf8(source, &all, none)
uts46_process_unicode(all.view(), 0, 0, is_label, to_ascii, &dest_string, options, info)
# destString.toUTF8(dest)
for code_point in dest_string.view():
encode_utf8(code_point, &dest)
if to_ascii and not is_label:
# length==labelStart==254 means that there is a trailing dot (ok) and
# destString is empty (do not index at 253-labelStart).
let length = label_start + dest_string.count
if length >= 254 and is_ascii_code_points(dest_string.view()) and (length > 254 or (label_start < 254 and dest_string.items[253 - label_start] != 0x2e)):
info.errors |= uidna_error_domain_name_too_long
if info.is_bidi and not info.failed and (info.errors & severe_errors) == 0 and (not info.is_ok_bidi or (label_start > 0 and not is_ascii_ok_bidi(source[..<label_start]))):
info.errors |= uidna_error_bidi
return dest
func is_ascii_code_points(text: const u32[]) -> bool:
for c in text:
if c > 0x7f:
return false
return true
## UTS46::processUnicode: `dest` holds the mapped ASCII prefix of the current label (up to
## mappingStart); the rest of `source` is mapped, normalized and processed label by label.
func uts46_process_unicode(source: const u32[], label_start_: usize, mapping_start: usize, is_label: bool, to_ascii: bool, dest: CodePointBuffer*, options: u32, info: IdnaInfo*):
var label_start = label_start_
if mapping_start == 0:
dest.count = 0
let normalized = uts46_normalize(source)
dest.append_all(normalized.view())
else:
# normalizeSecondAndAppend(dest, src.tempSubString(mappingStart)): the prefix is
# mapped ASCII, which the mapping keeps, so normalizing the whole is the same.
var whole = CodePointBuffer.create(dest.count + source.length + 1)
whole.append_all(dest.view())
whole.append_all(source[mapping_start..])
let normalized = uts46_normalize(whole.view())
dest.count = 0
dest.append_all(normalized.view())
var do_map_dev_chars = (options & uidna_nontransitional_to_ascii) == 0 if to_ascii else (options & uidna_nontransitional_to_unicode) == 0
var label_limit = label_start
while label_limit < dest.count:
let c = dest.items[label_limit]
if c == 0x2e and not is_label:
let label_length = label_limit - label_start
let new_length = uts46_process_label(dest, label_start, label_length, to_ascii, options, info)
info.errors |= info.label_errors
info.label_errors = 0
if info.failed:
return
label_start += new_length + 1
label_limit = label_start
continue
elif c < 0xdf:
# pass
_ = c
elif c <= 0x200d and (c == 0xdf or c == 0x3c2 or c >= 0x200c):
info.is_trans_diff = true
if do_map_dev_chars:
uts46_map_dev_chars(dest, label_start, label_limit)
# All deviation characters have been mapped, no need to check for them again.
do_map_dev_chars = false
# Do not increment labelLimit in case c was removed.
continue
label_limit += 1
# Permit an empty label at the end (0<labelStart==labelLimit==destLength is ok)
# but not an empty label elsewhere nor a completely empty domain name.
# processLabel() sets UIDNA_ERROR_EMPTY_LABEL when labelLength==0.
if label_start == 0 or label_start < label_limit:
_ = uts46_process_label(dest, label_start, label_limit - label_start, to_ascii, options, info)
info.errors |= info.label_errors
## UTS46::mapDevChars: transitional processing of ß, ς, ZWJ and ZWNJ from `mapping_start`.
func uts46_map_dev_chars(dest: CodePointBuffer*, label_start: usize, mapping_start: usize):
var mapped = CodePointBuffer.create(dest.count + 2)
mapped.append_all(dest.view()[..<mapping_start])
var did_map_dev_chars = false
for c in dest.view()[mapping_start..]:
if c == 0xdf:
# Map sharp s to ss.
did_map_dev_chars = true
mapped.append(0x73)
mapped.append(0x73)
elif c == 0x3c2:
# Map final sigma to nonfinal sigma.
did_map_dev_chars = true
mapped.append(0x3c3)
elif c == 0x200c or c == 0x200d:
# Ignore/remove ZWNJ and ZWJ.
did_map_dev_chars = true
else:
mapped.append(c)
dest.count = 0
if did_map_dev_chars:
# Mapping deviation characters might have resulted in an un-NFC string.
dest.append_all(mapped.view()[..<label_start])
let normalized = uts46_normalize(mapped.view()[label_start..])
dest.append_all(normalized.view())
else:
dest.append_all(mapped.view())
## Replace dest[start..start + length) with `label`.
func replace_label(dest: CodePointBuffer*, start: usize, length: usize, label: const u32[]):
var result = CodePointBuffer.create(dest.count - length + label.length + 1)
result.append_all(dest.view()[..<start])
result.append_all(label)
result.append_all(dest.view()[start + length..])
dest.count = 0
dest.append_all(result.view())
## UTS46::processLabel: validate and convert one label; answers its new length.
func uts46_process_label(dest: CodePointBuffer*, label_start_: usize, label_length_: usize, to_ascii: bool, options: u32, info: IdnaInfo*) -> usize:
let dest_label_start = label_start_
var dest_label_length = label_length_
var label_length = label_length_
var was_punycode = false
# The label being checked: a copy of dest's label, or the Punycode-decoded label.
var label = CodePointBuffer.create(label_length + 1)
label.append_all(dest.view()[label_start_..<label_start_ + label_length])
let first4 = label.view()
if label_length >= 4 and first4[0] == 0x78 and first4[1] == 0x6e and first4[2] == 0x2d and first4[3] == 0x2d:
# Label starts with "xn--", try to un-Punycode it.
# In IDNA2008, labels like "xn--" (decodes to an empty string) and
# "xn--ASCII-" (decodes to just "ASCII") fail the round-trip validation from
# comparing the ToUnicode input with the back-to-ToASCII output.
# They are alternate encodings of the respective ASCII labels.
# Ignore "xn---" here: It will fail Punycode.decode() which logically comes before
# the round-trip verification.
if label_length == 4 or (label_length > 5 and first4[label_length - 1] == 0x2d):
info.label_errors |= uidna_error_invalid_ace_label
return uts46_mark_bad_ace_label(dest, dest_label_start, dest_label_length, to_ascii, options, info)
was_punycode = true
var from_punycode = CodePointBuffer.create(1)
if let decoded_label = punycode_decode(first4[4..]):
from_punycode = decoded_label
else:
info.label_errors |= uidna_error_punycode
return uts46_mark_bad_ace_label(dest, dest_label_start, dest_label_length, to_ascii, options, info)
# Check for NFC, and for characters that are not
# valid or deviation characters according to the normalizer.
# If there is something wrong, then the string will change.
# Note that the normalizer passes through non-LDH ASCII and deviation characters.
# Deviation characters are ok in Punycode even in transitional processing.
# In the code further below, if we find non-LDH ASCII and we have UIDNA_USE_STD3_RULES
# then we will set UIDNA_ERROR_INVALID_ACE_LABEL there too.
let is_valid = uts46_is_normalized(from_punycode.view())
# Unicode 15.1 UTS #46:
# Added an additional condition in 4.1 Validity Criteria to
# disallow labels such as xn--xn---epa., which do not round-trip.
# --> Validity Criteria new criterion 4:
# If not CheckHyphens, the label must not begin with “xn--”.
let decoded = from_punycode.view()
let starts_with_xn = decoded.length >= 4 and decoded[0] == 0x78 and decoded[1] == 0x6e and decoded[2] == 0x2d and decoded[3] == 0x2d
if not is_valid or starts_with_xn:
info.label_errors |= uidna_error_invalid_ace_label
return uts46_mark_bad_ace_label(dest, dest_label_start, dest_label_length, to_ascii, options, info)
label = from_punycode
label_length = label.count
# Validity check
if label_length == 0:
info.label_errors |= uidna_error_empty_label
replace_label(dest, dest_label_start, dest_label_length, label.view())
return label_length
# labelLength>0
let s = label.items
if label_length >= 4 and s[2] == 0x2d and s[3] == 0x2d:
# label starts with "??--"
info.label_errors |= uidna_error_hyphen_3_4
if s[0] == 0x2d:
# label starts with "-"
info.label_errors |= uidna_error_leading_hyphen
if s[label_length - 1] == 0x2d:
# label ends with "-"
info.label_errors |= uidna_error_trailing_hyphen
# If the label was not a Punycode label, then it was the result of
# mapping, normalization and label segmentation.
# If the label was in Punycode, then we mapped it again above
# and checked its validity.
# Now we handle the STD3 restriction to LDH characters (if set)
# and we look for U+FFFD which indicates disallowed characters
# in a non-Punycode label or U+FFFD itself in a Punycode label.
# We also check for dots which can come from the input to a single-label function.
var ored_chars: u32 = 0
# If we enforce STD3 rules, then ASCII characters other than LDH and dot are disallowed.
let disallow_non_ldh_dot = (options & uidna_use_std3_rules) != 0
for index in 0..<label_length:
let c = s[index]
if c <= 0x7f:
if c == 0x2e:
info.label_errors |= uidna_error_label_has_dot
s[index] = 0xfffd
elif disallow_non_ldh_dot and ascii_data(c) < 0:
info.label_errors |= uidna_error_disallowed
s[index] = 0xfffd
else:
ored_chars |= c
if c == 0xfffd:
info.label_errors |= uidna_error_disallowed
# Check for a leading combining mark after other validity checks
# so that we don't report UIDNA_ERROR_DISALLOWED for the U+FFFD from here.
if (ucd_general_category_mask(s[0]) & u_gc_m_mask) != 0:
info.label_errors |= uidna_error_leading_combining_mark
s[0] = 0xfffd
let label_view = label.view()
if (info.label_errors & severe_errors) == 0:
# Do contextual checks only if we do not have U+FFFD from a severe error
# because U+FFFD can make these checks fail.
if (options & uidna_check_bidi) != 0 and (not info.is_bidi or info.is_ok_bidi):
uts46_check_label_bidi(label_view, info)
if (options & uidna_check_contextj) != 0 and (ored_chars & 0x200c) == 0x200c and not uts46_is_label_ok_context_j(label_view):
info.label_errors |= uidna_error_contextj
if to_ascii:
if was_punycode:
# Leave a Punycode label unchanged if it has no severe errors.
if dest_label_length > 63:
info.label_errors |= uidna_error_label_too_long
return dest_label_length
elif ored_chars >= 0x80:
# Contains non-ASCII characters.
var punycode = CodePointBuffer.create(label_length + 8)
# Write "xn--".
punycode.append(0x78)
punycode.append(0x6e)
punycode.append(0x2d)
punycode.append(0x2d)
if not punycode_encode(label_view, &punycode):
info.failed = true
return dest_label_length
if punycode.count > 63:
info.label_errors |= uidna_error_label_too_long
replace_label(dest, dest_label_start, dest_label_length, punycode.view())
return punycode.count
else:
# all-ASCII label
if label_length > 63:
info.label_errors |= uidna_error_label_too_long
else:
# If a Punycode label has severe errors,
# then leave it but make sure it does not look valid.
if was_punycode:
info.label_errors |= uidna_error_invalid_ace_label
return uts46_mark_bad_ace_label(dest, dest_label_start, dest_label_length, to_ascii, options, info)
replace_label(dest, dest_label_start, dest_label_length, label_view)
dest_label_length = label_length
return dest_label_length