Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-regex

Source of release 0.6.1

0.6.1 / src / unicode / properties.lucb

#==============================================================================================
#
#   properties - Character ranges for scripts and general categories
#
#   DESCRIPTION:
#       Port of QuickJS libunicode.c lines 1245-1458 and 1595-1623: unicode_find_name,
#       unicode_script, unicode_general_category1 and unicode_general_category. Names are
#       `str`; the C `const char *` names are ASCII.
#
#==============================================================================================

# mark: Names ==================================================================================

# char ranges for various unicode properties

# The index of the entry of `name_table` (NUL-separated entries of comma-separated aliases,
# ended by an empty entry) that has `name` as one of its aliases, or -1.
func unicode_find_name(name_table: const u8[], name: str) -> i32:
    var p: usize = 0
    var pos: i32 = 0
    let name_len = name.length
    while name_table[p] != 0:
        while true:
            # r = strchr(p, ','): the next comma of this entry, if any
            var r = p
            while name_table[r] != 0 and name_table[r] != ',':
                r += 1
            let has_comma = name_table[r] == ','
            let len = r - p
            if len == name_len and memcmp(&name_table[p], name.bytes.data, name_len) == 0:
                return pos
            p += len + 1
            if not has_comma:
                break
        pos += 1
    return -1

# mark: Scripts ================================================================================

# 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2
# if not found
## Add to `cr` (initialised and empty) the code points of the script `script_name` (a long
## or short name), with its Script_Extensions when `is_ext`; 0, -1 when memory runs out,
## -2 when the name is unknown.
pub func unicode_script(cr: CharRange*, script_name: str, is_ext: bool) -> i32:
    let script_idx = unicode_find_name(unicode_script_name_table, script_name)
    if script_idx < 0:
        return -2

    let is_common = (script_idx == unicode_script_common or script_idx == unicode_script_inherited)
    var cr1_s: CharRange
    var cr2_s: CharRange
    let cr2 = &cr2_s
    var cr1 = cr
    if is_ext:
        cr1 = &cr1_s
        cr_init(cr1, cr.mem_opaque, cr.realloc_func)
        cr_init(cr2, cr.mem_opaque, cr.realloc_func)
    # upstream's `fail:` frees and then `goto fail`s again, which never returns; here the
    # failure frees once and answers -1.
    if unicode_script1(cr, cr1, cr2, script_idx, is_ext, is_common) != 0:
        if is_ext:
            cr_free(cr1)
            cr_free(cr2)
        return -1
    return 0

# The body of unicode_script between its setup and its `fail:` label.
func unicode_script1(cr: CharRange*, cr1: CharRange*, cr2: CharRange*, script_idx: i32,
                     is_ext: bool, is_common: bool) -> i32:
    var p: const u8* = &unicode_script_table[0]
    var p_end = p + unicode_script_table.length
    var c: u32 = 0
    while p < p_end:
        let b: u32 = *p
        p = p + 1
        let run_type = b >> 7
        var n = b & 0x7f
        # (00..5F: the length is n itself)
        if n >= 96 and n < 112:
            n = (n - 96) << 8
            let e0: u32 = *p
            p = p + 1
            n |= e0
            n += 96
        elif n >= 112:
            n = (n - 112) << 16
            let e0: u32 = *p
            p = p + 1
            n |= e0 << 8
            let e1: u32 = *p
            p = p + 1
            n |= e1
            n += 96 + (1 << 12)
        let c1 = c + n + 1
        if run_type != 0:
            let v: u32 = *p
            p = p + 1
            if (i32)v == script_idx or script_idx == unicode_script_unknown:
                if cr_add_interval(cr1, c, c1) != 0:
                    return -1
        c = c1
    if script_idx == unicode_script_unknown:
        # Unknown is all the characters outside scripts
        if cr_invert(cr1) != 0:
            return -1

    if is_ext:
        # add the script extensions
        p = &unicode_script_ext_table[0]
        p_end = p + unicode_script_ext_table.length
        c = 0
        while p < p_end:
            let b: u32 = *p
            p = p + 1
            var n: u32 = 0
            if b < 128:
                n = b
            elif b < 128 + 64:
                n = (b - 128) << 8
                let e0: u32 = *p
                p = p + 1
                n |= e0
                n += 128
            else:
                n = (b - 128 - 64) << 16
                let e0: u32 = *p
                p = p + 1
                n |= e0 << 8
                let e1: u32 = *p
                p = p + 1
                n |= e1
                n += 128 + (1 << 14)
            let c1 = c + n + 1
            let v_len: usize = *p
            p = p + 1
            if is_common:
                if v_len != 0:
                    if cr_add_interval(cr2, c, c1) != 0:
                        return -1
            else:
                for i in 0..<v_len:
                    if (i32)p[i] == script_idx:
                        if cr_add_interval(cr2, c, c1) != 0:
                            return -1
                        break
            p = p + v_len
            c = c1
        if is_common:
            # remove all the characters with script extensions
            if cr_invert(cr2) != 0:
                return -1
            if cr_op(cr, cr_points(cr1), cr_points(cr2), cr_op_inter) != 0:
                return -1
        else:
            if cr_op(cr, cr_points(cr1), cr_points(cr2), cr_op_union) != 0:
                return -1
        cr_free(cr1)
        cr_free(cr2)
    return 0

# mark: General categories =====================================================================

# M(id): the bit of the general category `id` in a category mask.
func m(id: i32) -> u32:
    return 1u32 << (u32)id

func unicode_general_category1(cr: CharRange*, gc_mask: u32) -> i32:
    var p: const u8* = &unicode_gc_table[0]
    let p_end = p + unicode_gc_table.length
    var c: u32 = 0
    # Compressed range encoding:
    #    initial byte:
    #    bits 0..4: category number (special case 31)
    #    bits 5..7: range length (add 1)
    #    special case bits 5..7 == 7: read an extra byte
    #    - 00..7F: range length (add 7 + 1)
    #    - 80..BF: 6-bits plus extra byte for range length (add 7 + 128)
    #    - C0..FF: 6-bits plus 2 extra bytes for range length (add 7 + 128 + 16384)
    while p < p_end:
        let b: u32 = *p
        p = p + 1
        var n = b >> 5
        let v = b & 0x1f
        if n == 7:
            n = *p
            p = p + 1
            if n < 128:
                n += 7
            elif n < 128 + 64:
                n = (n - 128) << 8
                let e0: u32 = *p
                p = p + 1
                n |= e0
                n += 7 + 128
            else:
                n = (n - 128 - 64) << 16
                let e0: u32 = *p
                p = p + 1
                n |= e0 << 8
                let e1: u32 = *p
                p = p + 1
                n |= e1
                n += 7 + 128 + (1 << 14)
        var c0 = c
        c += n + 1
        var add_range = false  # the C `goto add_range`
        if v == 31:
            # run of Lu / Ll
            let lu_ll = gc_mask & (m(unicode_gc_lu) | m(unicode_gc_ll))
            if lu_ll != 0:
                if lu_ll == (m(unicode_gc_lu) | m(unicode_gc_ll)):
                    add_range = true
                else:
                    c0 += (u32)((gc_mask & m(unicode_gc_ll)) != 0)
                    while c0 < c:
                        if cr_add_interval(cr, c0, c0 + 1) != 0:
                            return -1
                        c0 += 2
        elif ((gc_mask >> v) & 1) != 0:
            add_range = true
        if add_range:
            if cr_add_interval(cr, c0, c) != 0:
                return -1
    return 0

let unicode_gc_mask_table: u32[8] = [
    (1u32 << unicode_gc_lu) | (1u32 << unicode_gc_ll) | (1u32 << unicode_gc_lt), # LC
    (1u32 << unicode_gc_lu) | (1u32 << unicode_gc_ll) | (1u32 << unicode_gc_lt) |
        (1u32 << unicode_gc_lm) | (1u32 << unicode_gc_lo), # L
    (1u32 << unicode_gc_mn) | (1u32 << unicode_gc_mc) | (1u32 << unicode_gc_me), # M
    (1u32 << unicode_gc_nd) | (1u32 << unicode_gc_nl) | (1u32 << unicode_gc_no), # N
    (1u32 << unicode_gc_sm) | (1u32 << unicode_gc_sc) | (1u32 << unicode_gc_sk) |
        (1u32 << unicode_gc_so), # S
    (1u32 << unicode_gc_pc) | (1u32 << unicode_gc_pd) | (1u32 << unicode_gc_ps) |
        (1u32 << unicode_gc_pe) | (1u32 << unicode_gc_pi) | (1u32 << unicode_gc_pf) |
        (1u32 << unicode_gc_po), # P
    (1u32 << unicode_gc_zs) | (1u32 << unicode_gc_zl) | (1u32 << unicode_gc_zp), # Z
    (1u32 << unicode_gc_cc) | (1u32 << unicode_gc_cf) | (1u32 << unicode_gc_cs) |
        (1u32 << unicode_gc_co) | (1u32 << unicode_gc_cn), # C
]

# 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2
# if not found
## Add to `cr` (initialised and empty) the code points of the general category `gc_name`
## (a long or short name, or a group such as `L`); 0, -1 when memory runs out, -2 when the
## name is unknown.
pub func unicode_general_category(cr: CharRange*, gc_name: str) -> i32:
    let gc_idx = unicode_find_name(unicode_gc_name_table, gc_name)
    if gc_idx < 0:
        return -2
    var gc_mask: u32 = 0
    if gc_idx <= unicode_gc_co:
        gc_mask = (u32)((u64)1 << (u64)gc_idx)
    else:
        gc_mask = unicode_gc_mask_table[(usize)(gc_idx - unicode_gc_lc)]
    return unicode_general_category1(cr, gc_mask)