dymokomi/luce-regex
Source of release 0.6.1
0.6.1 / src / unicode / properties.lucb
#==============================================================================================
#
# properties - Character ranges for scripts and general categories
#
# DESCRIPTION:
# Port of QuickJS libunicode.c lines 1245-1458 and 1595-1623: unicode_find_name,
# unicode_script, unicode_general_category1 and unicode_general_category. Names are
# `str`; the C `const char *` names are ASCII.
#
#==============================================================================================
# mark: Names ==================================================================================
# char ranges for various unicode properties
# The index of the entry of `name_table` (NUL-separated entries of comma-separated aliases,
# ended by an empty entry) that has `name` as one of its aliases, or -1.
func unicode_find_name(name_table: const u8[], name: str) -> i32:
var p: usize = 0
var pos: i32 = 0
let name_len = name.length
while name_table[p] != 0:
while true:
# r = strchr(p, ','): the next comma of this entry, if any
var r = p
while name_table[r] != 0 and name_table[r] != ',':
r += 1
let has_comma = name_table[r] == ','
let len = r - p
if len == name_len and memcmp(&name_table[p], name.bytes.data, name_len) == 0:
return pos
p += len + 1
if not has_comma:
break
pos += 1
return -1
# mark: Scripts ================================================================================
# 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2
# if not found
## Add to `cr` (initialised and empty) the code points of the script `script_name` (a long
## or short name), with its Script_Extensions when `is_ext`; 0, -1 when memory runs out,
## -2 when the name is unknown.
pub func unicode_script(cr: CharRange*, script_name: str, is_ext: bool) -> i32:
let script_idx = unicode_find_name(unicode_script_name_table, script_name)
if script_idx < 0:
return -2
let is_common = (script_idx == unicode_script_common or script_idx == unicode_script_inherited)
var cr1_s: CharRange
var cr2_s: CharRange
let cr2 = &cr2_s
var cr1 = cr
if is_ext:
cr1 = &cr1_s
cr_init(cr1, cr.mem_opaque, cr.realloc_func)
cr_init(cr2, cr.mem_opaque, cr.realloc_func)
# upstream's `fail:` frees and then `goto fail`s again, which never returns; here the
# failure frees once and answers -1.
if unicode_script1(cr, cr1, cr2, script_idx, is_ext, is_common) != 0:
if is_ext:
cr_free(cr1)
cr_free(cr2)
return -1
return 0
# The body of unicode_script between its setup and its `fail:` label.
func unicode_script1(cr: CharRange*, cr1: CharRange*, cr2: CharRange*, script_idx: i32,
is_ext: bool, is_common: bool) -> i32:
var p: const u8* = &unicode_script_table[0]
var p_end = p + unicode_script_table.length
var c: u32 = 0
while p < p_end:
let b: u32 = *p
p = p + 1
let run_type = b >> 7
var n = b & 0x7f
# (00..5F: the length is n itself)
if n >= 96 and n < 112:
n = (n - 96) << 8
let e0: u32 = *p
p = p + 1
n |= e0
n += 96
elif n >= 112:
n = (n - 112) << 16
let e0: u32 = *p
p = p + 1
n |= e0 << 8
let e1: u32 = *p
p = p + 1
n |= e1
n += 96 + (1 << 12)
let c1 = c + n + 1
if run_type != 0:
let v: u32 = *p
p = p + 1
if (i32)v == script_idx or script_idx == unicode_script_unknown:
if cr_add_interval(cr1, c, c1) != 0:
return -1
c = c1
if script_idx == unicode_script_unknown:
# Unknown is all the characters outside scripts
if cr_invert(cr1) != 0:
return -1
if is_ext:
# add the script extensions
p = &unicode_script_ext_table[0]
p_end = p + unicode_script_ext_table.length
c = 0
while p < p_end:
let b: u32 = *p
p = p + 1
var n: u32 = 0
if b < 128:
n = b
elif b < 128 + 64:
n = (b - 128) << 8
let e0: u32 = *p
p = p + 1
n |= e0
n += 128
else:
n = (b - 128 - 64) << 16
let e0: u32 = *p
p = p + 1
n |= e0 << 8
let e1: u32 = *p
p = p + 1
n |= e1
n += 128 + (1 << 14)
let c1 = c + n + 1
let v_len: usize = *p
p = p + 1
if is_common:
if v_len != 0:
if cr_add_interval(cr2, c, c1) != 0:
return -1
else:
for i in 0..<v_len:
if (i32)p[i] == script_idx:
if cr_add_interval(cr2, c, c1) != 0:
return -1
break
p = p + v_len
c = c1
if is_common:
# remove all the characters with script extensions
if cr_invert(cr2) != 0:
return -1
if cr_op(cr, cr_points(cr1), cr_points(cr2), cr_op_inter) != 0:
return -1
else:
if cr_op(cr, cr_points(cr1), cr_points(cr2), cr_op_union) != 0:
return -1
cr_free(cr1)
cr_free(cr2)
return 0
# mark: General categories =====================================================================
# M(id): the bit of the general category `id` in a category mask.
func m(id: i32) -> u32:
return 1u32 << (u32)id
func unicode_general_category1(cr: CharRange*, gc_mask: u32) -> i32:
var p: const u8* = &unicode_gc_table[0]
let p_end = p + unicode_gc_table.length
var c: u32 = 0
# Compressed range encoding:
# initial byte:
# bits 0..4: category number (special case 31)
# bits 5..7: range length (add 1)
# special case bits 5..7 == 7: read an extra byte
# - 00..7F: range length (add 7 + 1)
# - 80..BF: 6-bits plus extra byte for range length (add 7 + 128)
# - C0..FF: 6-bits plus 2 extra bytes for range length (add 7 + 128 + 16384)
while p < p_end:
let b: u32 = *p
p = p + 1
var n = b >> 5
let v = b & 0x1f
if n == 7:
n = *p
p = p + 1
if n < 128:
n += 7
elif n < 128 + 64:
n = (n - 128) << 8
let e0: u32 = *p
p = p + 1
n |= e0
n += 7 + 128
else:
n = (n - 128 - 64) << 16
let e0: u32 = *p
p = p + 1
n |= e0 << 8
let e1: u32 = *p
p = p + 1
n |= e1
n += 7 + 128 + (1 << 14)
var c0 = c
c += n + 1
var add_range = false # the C `goto add_range`
if v == 31:
# run of Lu / Ll
let lu_ll = gc_mask & (m(unicode_gc_lu) | m(unicode_gc_ll))
if lu_ll != 0:
if lu_ll == (m(unicode_gc_lu) | m(unicode_gc_ll)):
add_range = true
else:
c0 += (u32)((gc_mask & m(unicode_gc_ll)) != 0)
while c0 < c:
if cr_add_interval(cr, c0, c0 + 1) != 0:
return -1
c0 += 2
elif ((gc_mask >> v) & 1) != 0:
add_range = true
if add_range:
if cr_add_interval(cr, c0, c) != 0:
return -1
return 0
let unicode_gc_mask_table: u32[8] = [
(1u32 << unicode_gc_lu) | (1u32 << unicode_gc_ll) | (1u32 << unicode_gc_lt), # LC
(1u32 << unicode_gc_lu) | (1u32 << unicode_gc_ll) | (1u32 << unicode_gc_lt) |
(1u32 << unicode_gc_lm) | (1u32 << unicode_gc_lo), # L
(1u32 << unicode_gc_mn) | (1u32 << unicode_gc_mc) | (1u32 << unicode_gc_me), # M
(1u32 << unicode_gc_nd) | (1u32 << unicode_gc_nl) | (1u32 << unicode_gc_no), # N
(1u32 << unicode_gc_sm) | (1u32 << unicode_gc_sc) | (1u32 << unicode_gc_sk) |
(1u32 << unicode_gc_so), # S
(1u32 << unicode_gc_pc) | (1u32 << unicode_gc_pd) | (1u32 << unicode_gc_ps) |
(1u32 << unicode_gc_pe) | (1u32 << unicode_gc_pi) | (1u32 << unicode_gc_pf) |
(1u32 << unicode_gc_po), # P
(1u32 << unicode_gc_zs) | (1u32 << unicode_gc_zl) | (1u32 << unicode_gc_zp), # Z
(1u32 << unicode_gc_cc) | (1u32 << unicode_gc_cf) | (1u32 << unicode_gc_cs) |
(1u32 << unicode_gc_co) | (1u32 << unicode_gc_cn), # C
]
# 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2
# if not found
## Add to `cr` (initialised and empty) the code points of the general category `gc_name`
## (a long or short name, or a group such as `L`); 0, -1 when memory runs out, -2 when the
## name is unknown.
pub func unicode_general_category(cr: CharRange*, gc_name: str) -> i32:
let gc_idx = unicode_find_name(unicode_gc_name_table, gc_name)
if gc_idx < 0:
return -2
var gc_mask: u32 = 0
if gc_idx <= unicode_gc_co:
gc_mask = (u32)((u64)1 << (u64)gc_idx)
else:
gc_mask = unicode_gc_mask_table[(usize)(gc_idx - unicode_gc_lc)]
return unicode_general_category1(cr, gc_mask)