Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / web_unicode / idna_2.lucb

#==============================================================================================
#
#   idna_2 - UTS #46 validity checks and Punycode (ICU uts46.cpp, punycode.cpp)
#
#   DESCRIPTION:
#       The rest of the ICU code idna.lucb replaces: UTS46::markBadACELabel, checkLabelBiDi,
#       isASCIIOkBiDi and isLabelOkContextJ of ICU 78's uts46.cpp, and u_strToPunycode and
#       u_strFromPunycode of punycode.cpp, © Unicode, Inc. and others, Unicode License v3.
#
#==============================================================================================

# mark: UTS46 validity (ICU uts46.cpp) ==========================================================

## UTS46::markBadACELabel: make sure an ACE label does not look valid. Append U+FFFD if the
## label has only LDH characters; with UIDNA_USE_STD3_RULES, also replace disallowed ASCII
## characters with U+FFFD.
func uts46_mark_bad_ace_label(dest: CodePointBuffer*, label_start: usize, label_length_: usize, to_ascii: bool, options: u32, info: IdnaInfo*) -> usize:
    var label_length = label_length_
    let disallow_non_ldh_dot = (options & uidna_use_std3_rules) != 0
    var is_ascii = true
    var only_ldh = true
    # Start after the initial "xn--".
    for index in label_start + 4..<label_start + label_length:
        let c = dest.items[index]
        if c <= 0x7f:
            if c == 0x2e:
                info.label_errors |= uidna_error_label_has_dot
                dest.items[index] = 0xfffd
                is_ascii = false
                only_ldh = false
            elif ascii_data(c) < 0:
                only_ldh = false
                if disallow_non_ldh_dot:
                    dest.items[index] = 0xfffd
                    is_ascii = false
        else:
            is_ascii = false
            only_ldh = false
    if only_ldh:
        let marker: u32[1] = [0xfffd]
        replace_label(dest, label_start + label_length, 0, marker)
        label_length += 1
    else:
        if to_ascii and is_ascii and label_length > 63:
            info.label_errors |= uidna_error_label_too_long
    return label_length

# UCharDirection masks of the BiDi rule.
func direction_mask(direction: u32) -> u32:
    return (u32)1 << direction

let l_mask: u32 = 1
let r_al_mask: u32 = (1 << 1) | (1 << 13)
let l_r_al_mask: u32 = l_mask | r_al_mask
let r_al_an_mask: u32 = r_al_mask | (1 << 5)
let en_an_mask: u32 = (1 << 2) | (1 << 5)
let r_al_en_an_mask: u32 = r_al_mask | en_an_mask
let l_en_mask: u32 = l_mask | (1 << 2)
# ES, CS, ET, ON, BN, NSM
let es_cs_et_on_bn_nsm_mask: u32 = (1 << 3) | (1 << 6) | (1 << 4) | (1 << 10) | (1 << 18) | (1 << 17)
let l_en_es_cs_et_on_bn_nsm_mask: u32 = l_en_mask | es_cs_et_on_bn_nsm_mask
let r_al_an_en_es_cs_et_on_bn_nsm_mask: u32 = r_al_mask | en_an_mask | es_cs_et_on_bn_nsm_mask

## UTS46::checkLabelBiDi.
# We scan the whole label and check both for whether it contains RTL characters
# and whether it passes the BiDi Rule.
# In a BiDi domain name, all labels must pass the BiDi Rule, but we might find
# that a domain name is a BiDi domain name (has an RTL label) only after
# processing several earlier labels.
func uts46_check_label_bidi(label: const u32[], info: IdnaInfo*):
    # IDNA2008 BiDi rule
    # Get the directionality of the first character.
    var i: usize = 1
    let first_mask = direction_mask(ucd_bidi_class(label[0]))
    # 1. The first character must be a character with BIDI property L, R
    # or AL.  If it has the R or AL property, it is an RTL label; if it
    # has the L property, it is an LTR label.
    if (first_mask & ~l_r_al_mask) != 0:
        info.is_ok_bidi = false
    # Get the directionality of the last non-NSM character.
    var last_mask: u32 = 0
    var label_length = label.length
    while true:
        if i >= label_length:
            last_mask = first_mask
            break
        label_length -= 1
        let direction = ucd_bidi_class(label[label_length])
        if direction != 17:
            last_mask = direction_mask(direction)
            break
    # 3. In an RTL label, the end of the label must be a character with
    # BIDI property R, AL, EN or AN, followed by zero or more
    # characters with BIDI property NSM.
    # 6. In an LTR label, the end of the label must be a character with
    # BIDI property L or EN, followed by zero or more characters with
    # BIDI property NSM.
    let bad_end = (last_mask & ~l_en_mask) != 0 if (first_mask & l_mask) != 0 else (last_mask & ~r_al_en_an_mask) != 0
    if bad_end:
        info.is_ok_bidi = false
    # Add the directionalities of the intervening characters.
    var mask = first_mask | last_mask
    while i < label_length:
        mask |= direction_mask(ucd_bidi_class(label[i]))
        i += 1
    if (first_mask & l_mask) != 0:
        # 5. In an LTR label, only characters with the BIDI properties L, EN,
        # ES, CS, ET, ON, BN and NSM are allowed.
        if (mask & ~l_en_es_cs_et_on_bn_nsm_mask) != 0:
            info.is_ok_bidi = false
    else:
        # 2. In an RTL label, only characters with the BIDI properties R, AL,
        # AN, EN, ES, CS, ET, ON, BN and NSM are allowed.
        if (mask & ~r_al_an_en_es_cs_et_on_bn_nsm_mask) != 0:
            info.is_ok_bidi = false
        # 4. In an RTL label, if an EN is present, no AN may be present, and
        # vice versa.
        if (mask & en_an_mask) == en_an_mask:
            info.is_ok_bidi = false
    # An RTL label is a label that contains at least one character of type
    # R, AL or AN. [...]
    # A "BIDI domain name" is a domain name that contains at least one RTL
    # label. [...]
    # The following rule, consisting of six conditions, applies to labels
    # in BIDI domain names.
    if (mask & r_al_an_mask) != 0:
        info.is_bidi = true

## isASCIIOkBiDi, the UTF-8 version, called for the source ASCII prefix (which can contain
## uppercase A-Z). s[length-1] must be the trailing dot.
# IDNA2008 BiDi rule, parts relevant to ASCII labels:
# 1. The first character must be a character with BIDI property L [...]
# 5. In an LTR label, only characters with the BIDI properties L, EN,
# ES, CS, ET, ON, BN and NSM are allowed.
# 6. In an LTR label, the end of the label must be a character with
# BIDI property L or EN [...]
func is_ascii_ok_bidi(s: const u8[]) -> bool:
    var label_start: usize = 0
    for i in 0..<s.length:
        var c = s[i]
        if c == 0x2e:
            # dot
            if i > label_start:
                c = s[i - 1]
                if not (0x61 <= c and c <= 0x7a) and not (0x41 <= c and c <= 0x5a) and not (0x30 <= c and c <= 0x39):
                    # Last character in the label is not an L or EN.
                    return false
            label_start = i + 1
        elif i == label_start:
            if not (0x61 <= c and c <= 0x7a) and not (0x41 <= c and c <= 0x5a):
                # First character in the label is not an L.
                return false
        else:
            if c <= 0x20 and (c >= 0x1c or (9 <= c and c <= 0xd)):
                # Intermediate character in the label is a B, S or WS.
                return false
    return true

# UJoiningType values.
let u_jt_dual_joining: u32 = 2
let u_jt_left_joining: u32 = 3
let u_jt_right_joining: u32 = 4
let u_jt_transparent: u32 = 5

## UTS46::isLabelOkContextJ.
func uts46_is_label_ok_context_j(label: const u32[]) -> bool:
    # [IDNA2008-Tables]
    # 200C..200D  ; CONTEXTJ    # ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER
    for i in 0..<label.length:
        if label[i] == 0x200c:
            # Appendix A.1. ZERO WIDTH NON-JOINER
            # Rule Set:
            #  False;
            #  If Canonical_Combining_Class(Before(cp)) .eq.  Virama Then True;
            #  If RegExpMatch((Joining_Type:{L,D})(Joining_Type:T)*‌
            #     (Joining_Type:T)*(Joining_Type:{R,D})) Then True;
            if i == 0:
                return false
            var j = i - 1
            var c = label[j]
            if ucd_combining_class(c) == 9:
                continue
            # check precontext (Joining_Type:{L,D})(Joining_Type:T)*
            while true:
                let type_ = ucd_joining_type(c)
                if type_ == u_jt_transparent:
                    if j == 0:
                        return false
                    j -= 1
                    c = label[j]
                elif type_ == u_jt_left_joining or type_ == u_jt_dual_joining:
                    # precontext fulfilled
                    break
                else:
                    return false
            # check postcontext (Joining_Type:T)*(Joining_Type:{R,D})
            j = i + 1
            while true:
                if j == label.length:
                    return false
                let type_ = ucd_joining_type(label[j])
                j += 1
                if type_ == u_jt_transparent:
                    # just skip this character
                    continue
                elif type_ == u_jt_right_joining or type_ == u_jt_dual_joining:
                    # postcontext fulfilled
                    break
                else:
                    return false
        elif label[i] == 0x200d:
            # Appendix A.2. ZERO WIDTH JOINER (U+200D)
            # Rule Set:
            #  False;
            #  If Canonical_Combining_Class(Before(cp)) .eq.  Virama Then True;
            if i == 0:
                return false
            if ucd_combining_class(label[i - 1]) != 9:
                return false
    return true

# mark: Punycode (ICU punycode.cpp) =============================================================

let punycode_base: i64 = 36
let punycode_tmin: i64 = 1
let punycode_tmax: i64 = 26
let punycode_skew: i64 = 38
let punycode_damp: i64 = 700
let punycode_initial_bias: i64 = 72
let punycode_initial_n: i64 = 0x80
# ICU-13727: Limit input length for n^2 algorithm
# where well-formed strings are at most 59 characters long.
let punycode_encode_max_code_units: usize = 1000
let punycode_decode_max_chars: usize = 2000

## digitToBasic, lowercase: 0..25 map to ASCII a..z, 26..35 map to ASCII 0..9.
func punycode_digit_to_basic(digit: i64) -> u32:
    if digit < 26:
        return (u32)(0x61 + digit)
    return (u32)(0x30 - 26 + digit)

## decodeDigit: the numeric value of a basic code point, or -1.
func punycode_decode_digit(cp: u32) -> i64:
    if cp <= 0x5a:
        if cp <= 0x39:
            if cp < 0x30:
                return -1
            # 0..9 -> 26..35
            return (i64)(cp - 0x30) + 26
        if cp < 0x41:
            return -1
        # A-Z -> 0..25
        return (i64)(cp - 0x41)
    if cp >= 0x61 and cp <= 0x7a:
        # a..z -> 0..25
        return (i64)(cp - 0x61)
    return -1

## Bias adaptation function.
func punycode_adapt_bias(delta_: i64, length: i64, first_time: bool) -> i64:
    var delta = delta_ // punycode_damp if first_time else delta_ // 2
    delta += delta // length
    var count: i64 = 0
    while delta > ((punycode_base - punycode_tmin) * punycode_tmax) // 2:
        delta = delta // (punycode_base - punycode_tmin)
        count += punycode_base
    return count + ((punycode_base - punycode_tmin + 1) * delta) // (delta + punycode_skew)

## u_strToPunycode without case flags, appending to `dest`; false on ICU's errors
## (U_INPUT_TOO_LONG_ERROR, U_INTERNAL_PROGRAM_ERROR).
func punycode_encode(source: const u32[], dest: CodePointBuffer*) -> bool:
    var utf16_length: usize = 0
    for c in source:
        utf16_length += 2 if c >= 0x10000 else 1
    if utf16_length > punycode_encode_max_code_units:
        return false
    # Handle the basic code points.
    var basic_length: i64 = 0
    for c in source:
        if c < 0x80:
            dest.append(c)
            basic_length += 1
    # Finish the basic string - if it is not empty - with a delimiter.
    if basic_length > 0:
        dest.append(0x2d)
    let src_cp_count = (i64)source.length
    var n = punycode_initial_n
    var delta: i64 = 0
    var bias = punycode_initial_bias
    var handled_cp_count = basic_length
    while handled_cp_count < src_cp_count:
        # All non-basic code points < n have been handled already. Find the next larger one:
        var m: i64 = 0x7fffffff
        for c in source:
            let q = (i64)c
            if n <= q and q < m:
                m = q
        # Increase delta enough to advance the decoder's <n,i> state to <m,0>, but guard
        # against overflow:
        if m - n > (0x7fffffff - handled_cp_count - delta) // (handled_cp_count + 1):
            return false
        delta += (m - n) * (handled_cp_count + 1)
        n = m
        # Encode a sequence of same code points n
        for c in source:
            let q_value = (i64)c
            if q_value < n:
                delta += 1
            elif q_value == n:
                # Represent delta as a generalized variable-length integer:
                var q = delta
                var k = punycode_base
                while true:
                    var t = k - bias
                    if t < punycode_tmin:
                        t = punycode_tmin
                    elif k >= bias + punycode_tmax:
                        t = punycode_tmax
                    if q < t:
                        break
                    dest.append(punycode_digit_to_basic(t + (q - t) % (punycode_base - t)))
                    q = (q - t) // (punycode_base - t)
                    k += punycode_base
                dest.append(punycode_digit_to_basic(q))
                bias = punycode_adapt_bias(delta, handled_cp_count + 1, handled_cp_count == basic_length)
                delta = 0
                handled_cp_count += 1
        delta += 1
        n += 1
    return true

## u_strFromPunycode without case flags: the decoded code points, or none on an error.
func punycode_decode(source: const u32[]) -> CodePointBuffer?:
    let src_length = source.length
    if src_length > punycode_decode_max_chars:
        return none
    # Handle the basic code points: let basicLength be the number of input code points
    # before the last delimiter, or 0 if there is none, then copy the first basicLength
    # code points to the output.
    var j = src_length
    while j > 0:
        j -= 1
        if source[j] == 0x2d:
            break
    let basic_length = j
    var dest = CodePointBuffer.create(src_length + 1)
    for index in 0..<basic_length:
        let b = source[index]
        if b >= 0x80:
            return none
        dest.append(b)
    # Initialize the state:
    var n = punycode_initial_n
    var i: i64 = 0
    var bias = punycode_initial_bias
    var dest_cp_count = (i64)basic_length
    # Main decoding loop: start just after the last delimiter if any basic code points
    # were copied; start at the beginning otherwise.
    var input = basic_length + 1 if basic_length > 0 else 0
    while input < src_length:
        # Decode a generalized variable-length integer into delta, which gets added to i.
        let oldi = i
        var w: i64 = 1
        var k = punycode_base
        while true:
            if input >= src_length:
                return none
            let digit = punycode_decode_digit(source[input])
            input += 1
            if digit < 0:
                return none
            if digit > (0x7fffffff - i) // w:
                # integer overflow
                return none
            i += digit * w
            var t = k - bias
            if t < punycode_tmin:
                t = punycode_tmin
            elif k >= bias + punycode_tmax:
                t = punycode_tmax
            if digit < t:
                break
            if w > 0x7fffffff // (punycode_base - t):
                # integer overflow
                return none
            w *= punycode_base - t
            k += punycode_base
        dest_cp_count += 1
        bias = punycode_adapt_bias(i - oldi, dest_cp_count, oldi == 0)
        # i was supposed to wrap around from (incremented) destCPCount to 0, incrementing n
        # each time, so we'll fix that now:
        if i // dest_cp_count > 0x7fffffff - n:
            # integer overflow
            return none
        n += i // dest_cp_count
        i = i % dest_cp_count
        if n > 0x10ffff or (n >= 0xd800 and n <= 0xdfff):
            # Unicode code point overflow
            return none
        # Insert n at position i of the output.
        dest.append(0)
        var position = dest.count - 1
        while position > (usize)i:
            dest.items[position] = dest.items[position - 1]
            position -= 1
        dest.items[(usize)i] = (u32)n
        i += 1
    return dest