dymokomi/luce-regex
Source of release 0.6.1
0.6.1 / src / unicode / tests.lucb
#==============================================================================================
#
# tests - Unit tests of the Unicode utilities against upstream libunicode.c
#
# DESCRIPTION:
# Every expectation comes from upstream libunicode.c (tests_expect_table.lucb): hashes
# of each lookup over all code points, pinned case conversions of the special cases,
# normalization of tricky sequences, and the ranges of every general category,
# script, binary property and sequence property. The character range operations are
# pinned by hand.
#
#==============================================================================================
# mark: Helpers ================================================================================
let fnv_offset: u32 = 2166136261
let code_point_end: u32 = 0x110000
func mix(h: u32, v: u32) -> u32:
return (h ^ v) *% 16777619
func range_hash(cr: const CharRange*) -> u32:
var h = fnv_offset
for v in cr_points(cr):
h = mix(h, v)
return h
# The first alias of entry `idx` of a NUL-separated name table.
func first_alias(table: const u8[], idx: i32) -> str:
var p: usize = 0
for i: i32 in 0..<idx:
while table[p] != 0:
p += 1
p += 1
var e = p
while table[e] != 0 and table[e] != 0x2c:
e += 1
return (str)table[p..<e]
func name_count(table: const u8[]) -> i32:
var p: usize = 0
var n: i32 = 0
while table[p] != 0:
while table[p] != 0:
p += 1
p += 1
n += 1
return n
# Compare `cr` and the status `ret` with a row (ret, len, hash, first and last interval).
func check_range_row(row: const u32[], ret: i32, cr: const CharRange*, what: str):
let points = cr_points(cr)
assert(row[0] == (u32)ret, what)
assert(row[1] == (u32)cr.len, what)
assert(row[2] == range_hash(cr), what)
if cr.len > 0:
assert(row[3] == points[0] and row[4] == points[1], what)
assert(row[5] == points[points.length - 2] and row[6] == points[points.length - 1], what)
func range_of(values: const u32[]) -> CharRange:
var cr: CharRange
cr_init(&cr, none, none)
for v in values:
assert(cr_add_point(&cr, v) == 0)
return cr
func range_is(cr: const CharRange*, values: const u32[]) -> bool:
let points = cr_points(cr)
if points.length != values.length:
return false
for i in 0..<values.length:
if points[i] != values[i]:
return false
return true
# mark: Case conversion ========================================================================
test "lre_case_conv over every code point":
let expected: u32[3] = [expect_case_conv0, expect_case_conv1, expect_case_conv2]
var res: u32[3]
for conv_type: i32 in 0..<3:
var h = fnv_offset
var c: u32 = 0
while c < code_point_end:
let l = lre_case_conv(res, c, conv_type)
h = mix(h, (u32)l)
for i in 0..<(usize)l:
h = mix(h, res[i])
c += 1
assert(h == expected[(usize)conv_type])
test "lre_canonicalize over every code point":
for mode in 0..<2:
var h = fnv_offset
var c: u32 = 0
while c < code_point_end:
h = mix(h, (u32)lre_canonicalize(c, mode == 1))
c += 1
assert(h == (expect_canonicalize0 if mode == 0 else expect_canonicalize1))
test "case conversion special cases":
# ß, İ, ı, ʼn, ſ, the DŽ titlecase triples, ǰ, ΐ, final sigma, ϐ, ϴ, և, Georgian,
# Cherokee, Greek extended with ypogegrammeni, K and Å signs, ff..st, Deseret, Osage,
# Old Hungarian, Adlam and the table ends
let rows = expect_case_points
var res: u32[3]
var r: usize = 0
while r < rows.length:
let c = rows[r]
for conv_type: i32 in 0..<3:
res = [0, 0, 0]
let at = r + 1 + (usize)conv_type * 4
let l = lre_case_conv(res, c, conv_type)
assert((u32)l == rows[at], "case length")
assert(res[0] == rows[at + 1], "case r0")
if l > 1:
assert(res[1] == rows[at + 2], "case r1")
if l > 2:
assert(res[2] == rows[at + 3], "case r2")
assert((u32)lre_canonicalize(c, false) == rows[r + 13], "canonicalize legacy")
assert((u32)lre_canonicalize(c, true) == rows[r + 14], "canonicalize unicode")
r += 15
test "case conversion examples":
var res: u32[3]
assert(lre_case_conv(res, 0xdf, 0) == 2 and res[0] == 0x53 and res[1] == 0x53)
assert(lre_case_conv(res, 0x130, 1) == 2 and res[0] == 0x69 and res[1] == 0x307)
assert(lre_case_conv(res, 0x3a3, 1) == 1 and res[0] == 0x3c3)
assert(lre_case_conv(res, 0x3c2, 0) == 1 and res[0] == 0x3a3)
assert(lre_case_conv(res, 0x3c2, 2) == 1 and res[0] == 0x3c3)
assert(lre_case_conv(res, 0x390, 0) == 3 and res[0] == 0x399 and res[1] == 0x308 and res[2] == 0x301)
assert(lre_case_conv(res, 0x13a0, 2) == 1 and res[0] == 0x13a0)
assert(lre_case_conv(res, 0xab70, 0) == 1 and res[0] == 0x13a0)
assert(lre_case_conv(res, 0x10400, 1) == 1 and res[0] == 0x10428)
assert(lre_case_conv(res, 0x1f80, 0) == 2 and res[0] == 0x1f08 and res[1] == 0x399)
assert(lre_canonicalize(0x17f, true) == 0x73)
assert(lre_canonicalize(0x17f, false) == 0x17f)
assert(lre_canonicalize(0x131, false) == 0x131)
assert(lre_canonicalize(0xfb06, true) == 0xfb05)
assert(lre_canonicalize(0x61, false) == 0x41)
assert(lre_canonicalize(0x41, true) == 0x61)
# mark: Predicates =============================================================================
test "predicates over every code point":
var cased = fnv_offset
var ignorable = fnv_offset
var id_start = fnv_offset
var id_continue = fnv_offset
var space = fnv_offset
var ident_first = fnv_offset
var ident_next = fnv_offset
var c: u32 = 0
while c < code_point_end:
cased = mix(cased, (u32)lre_is_cased(c))
ignorable = mix(ignorable, (u32)lre_is_case_ignorable(c))
id_start = mix(id_start, (u32)lre_is_id_start(c))
id_continue = mix(id_continue, (u32)lre_is_id_continue(c))
space = mix(space, (u32)lre_is_space(c))
ident_first = mix(ident_first, (u32)lre_js_is_ident_first(c))
ident_next = mix(ident_next, (u32)lre_js_is_ident_next(c))
c += 1
assert(cased == expect_is_cased)
assert(ignorable == expect_is_case_ignorable)
assert(id_start == expect_is_id_start)
assert(id_continue == expect_is_id_continue)
assert(space == expect_is_space)
assert(ident_first == expect_js_is_ident_first)
assert(ident_next == expect_js_is_ident_next)
test "byte classes":
var h = fnv_offset
for b: usize in 0..<256:
let c = (u8)b
h = mix(h, (u32)lre_is_id_start_byte(c))
h = mix(h, (u32)lre_is_id_continue_byte(c))
h = mix(h, (u32)lre_is_word_byte(c))
h = mix(h, lre_ctype_bits[b])
assert(h == expect_byte_classes)
assert(lre_is_space(0x2028) and lre_is_space(0xfeff) and lre_is_space(0xa0))
assert(not lre_is_space(0x200b) and not lre_is_space(0x78))
assert(lre_js_is_ident_next(0x200c) and not lre_js_is_ident_first(0x200c))
assert(lre_js_is_ident_first(0x24) and not lre_js_is_ident_first(0x31))
# mark: Normalization ==========================================================================
test "combining classes and decompositions over every code point":
var cc = fnv_offset
var d0 = fnv_offset
var d1 = fnv_offset
var res: u32[18]
var c: u32 = 0
while c < code_point_end:
cc = mix(cc, (u32)unicode_get_cc(c))
var l = unicode_decomp_char(res, c, false)
d0 = mix(d0, (u32)l)
for i in 0..<(usize)l:
d0 = mix(d0, res[i])
l = unicode_decomp_char(res, c, true)
d1 = mix(d1, (u32)l)
for i in 0..<(usize)l:
d1 = mix(d1, res[i])
c += 1
assert(cc == expect_cc)
assert(d0 == expect_decomp0)
assert(d1 == expect_decomp1)
test "unicode_normalize of every code point":
let expected: u32[4] = [expect_normalize0, expect_normalize1, expect_normalize2,
expect_normalize3]
for n_type: i32 in 0..<4:
var h = fnv_offset
var c: u32 = 0
while c < code_point_end:
let input: u32[1] = [c]
var out: u32*? = none
let l = unicode_normalize(&out, input, n_type, none, none)
assert(l >= 0)
h = mix(h, (u32)l)
let buf = out else trap("no output")
for i in 0..<(usize)l:
h = mix(h, buf[i])
c_free(buf)
c += 1
assert(h == expected[(usize)n_type])
test "unicode_normalize of tricky sequences":
# Hangul syllables and jamo, canonical ordering of combining marks, singletons (Ω, Å),
# composition exclusions, compatibility forms (fi, ①, ½, ㌀, ﷺ, halfwidth kana)
let rows = expect_normalize_sequences
var r: usize = 0
var count = 0
while r < rows.length:
let in_len = (usize)rows[r]
let input = rows[r + 1..<r + 1 + in_len]
r += 1 + in_len
for n_type: i32 in 0..<4:
var out: u32*? = none
let l = unicode_normalize(&out, input, n_type, none, none)
assert((u32)l == rows[r], "normalized length")
if l > 0:
let buf = out else trap("no output")
for i in 0..<(usize)l:
assert(buf[i] == rows[r + 1 + i], "normalized code point")
if let buf = out:
c_free(buf)
r += 1 + (usize)rows[r]
count += 1
assert(count == 40)
test "unicode_normalize examples":
var out: u32*? = none
let hangul: u32[3] = [0x1100, 0x1161, 0x11a8]
assert(unicode_normalize(&out, hangul, unicode_nfc, none, none) == 1)
var buf = out else trap("no output")
assert(buf[0] == 0xac01)
c_free(buf)
let marks: u32[3] = [0x65, 0x301, 0x327]
assert(unicode_normalize(&out, marks, unicode_nfd, none, none) == 3)
buf = out else trap("no output")
assert(buf[0] == 0x65 and buf[1] == 0x327 and buf[2] == 0x301)
c_free(buf)
let empty: u32[1] = [0]
assert(unicode_normalize(&out, empty[0..<0], unicode_nfkd, none, none) == 0)
# mark: Character ranges =======================================================================
test "cr_op on small ranges":
var a = range_of([0x10, 0x20, 0x30, 0x40])
let b: u32[2] = [0x18, 0x38]
var r: CharRange
cr_init(&r, none, none)
assert(cr_op(&r, cr_points(&a), b, cr_op_union) == 0)
assert(range_is(&r, [0x10, 0x40]))
r.len = 0
assert(cr_op(&r, cr_points(&a), b, cr_op_inter) == 0)
assert(range_is(&r, [0x18, 0x20, 0x30, 0x38]))
r.len = 0
assert(cr_op(&r, cr_points(&a), b, cr_op_xor) == 0)
assert(range_is(&r, [0x10, 0x18, 0x20, 0x30, 0x38, 0x40]))
r.len = 0
assert(cr_op(&r, cr_points(&a), b, cr_op_sub) == 0)
assert(range_is(&r, [0x10, 0x18, 0x38, 0x40]))
assert(cr_op1(&r, b, cr_op_union) == 0)
assert(range_is(&r, [0x10, 0x40]))
cr_free(&r)
cr_free(&a)
test "cr_invert, cr_union_interval, cr_copy and cr_compress":
var a = range_of([0x10, 0x20, 0x30, 0x40])
assert(cr_invert(&a) == 0)
assert(range_is(&a, [0, 0x10, 0x20, 0x30, 0x40, 0xffffffff]))
assert(cr_invert(&a) == 0)
assert(range_is(&a, [0x10, 0x20, 0x30, 0x40]))
assert(cr_union_interval(&a, 0x20, 0x2f) == 0)
assert(range_is(&a, [0x10, 0x40]))
assert(cr_union_interval(&a, 0x50, 0x50) == 0)
assert(range_is(&a, [0x10, 0x40, 0x50, 0x51]))
var b: CharRange
cr_init(&b, none, none)
assert(cr_copy(&b, &a) == 0)
assert(range_is(&b, [0x10, 0x40, 0x50, 0x51]))
var c = range_of([1, 1, 2, 3, 3, 5, 7, 7])
cr_compress(&c)
assert(range_is(&c, [2, 5]))
var empty: CharRange
cr_init(&empty, none, none)
assert(cr_invert(&empty) == 0)
assert(range_is(&empty, [0, 0xffffffff]))
cr_free(&empty)
cr_free(&c)
cr_free(&b)
cr_free(&a)
test "cr_regexp_canonicalize on small ranges":
var a = range_of([0x61, 0x7a + 1])
assert(cr_regexp_canonicalize(&a, false) == 0)
assert(range_is(&a, [0x41, 0x5a + 1]))
cr_free(&a)
var k = range_of([0x212a, 0x212b])
assert(cr_regexp_canonicalize(&k, true) == 0)
assert(range_is(&k, [0x6b, 0x6b + 1]))
cr_free(&k)
# A realloc hook that counts its calls in `opaque` and fails once the count is reached.
struct AllocBudget:
var calls: i32
var limit: i32
func budget_realloc(opaque: void*?, ptr: void*?, size: usize) -> void*?:
let budget = (AllocBudget*)(opaque else trap("no budget"))
budget.calls += 1
if size != 0 and budget.calls > budget.limit:
return none
return c_realloc(ptr, size)
test "allocation failures answer -1":
var budget = AllocBudget(calls = 0, limit = 1)
var cr: CharRange
cr_init(&cr, &budget, budget_realloc)
assert(cr_add_interval(&cr, 1, 2) == 0)
assert(cr_add_interval(&cr, 3, 4) == -1)
assert(budget.calls == 2)
cr_free(&cr)
budget = AllocBudget(calls = 0, limit = 0)
cr_init(&cr, &budget, budget_realloc)
assert(unicode_general_category(&cr, "L") == -1)
assert(unicode_prop(&cr, "Alphabetic") == -1)
assert(unicode_script(&cr, "Greek", true) == -1)
var out: u32*? = none
let input: u32[2] = [0x41, 0x30a]
assert(unicode_normalize(&out, input, unicode_nfc, &budget, budget_realloc) == -1)
assert(out == none)
# mark: Properties =============================================================================
test "general categories":
let rows = expect_general_categories
let n = name_count(unicode_gc_name_table)
assert((usize)n * 13 == rows.length)
for j: i32 in 0..<n:
let row = rows[(usize)j * 13..<(usize)j * 13 + 13]
let name = first_alias(unicode_gc_name_table, j)
var cr: CharRange
cr_init(&cr, none, none)
let ret = unicode_general_category(&cr, name)
check_range_row(row, ret, &cr, name)
var c2: CharRange
cr_init(&c2, none, none)
assert(cr_copy(&c2, &cr) == 0)
assert(cr_regexp_canonicalize(&cr, false) == 0)
assert(row[7] == (u32)cr.len and row[8] == range_hash(&cr), name)
assert(cr_regexp_canonicalize(&c2, true) == 0)
assert(row[9] == (u32)c2.len and row[10] == range_hash(&c2), name)
assert(cr_invert(&c2) == 0)
assert(row[11] == (u32)c2.len and row[12] == range_hash(&c2), name)
cr_free(&cr)
cr_free(&c2)
test "scripts with and without extensions":
let rows = expect_scripts
let n = name_count(unicode_script_name_table)
assert((usize)n * 14 == rows.length)
for j: i32 in 0..<n:
let name = first_alias(unicode_script_name_table, j)
for ext: usize in 0..<2:
let at = (usize)j * 14 + ext * 7
var cr: CharRange
cr_init(&cr, none, none)
let ret = unicode_script(&cr, name, ext == 1)
check_range_row(rows[at..<at + 7], ret, &cr, name)
cr_free(&cr)
test "binary properties":
let rows = expect_properties
let n = name_count(unicode_prop_name_table)
assert((usize)n * 7 == rows.length)
for j: i32 in 0..<n:
let name = first_alias(unicode_prop_name_table, j)
var cr: CharRange
cr_init(&cr, none, none)
let ret = unicode_prop(&cr, name)
check_range_row(rows[(usize)j * 7..<(usize)j * 7 + 7], ret, &cr, name)
cr_free(&cr)
test "aliases and unknown names":
var a: CharRange
var b: CharRange
cr_init(&a, none, none)
cr_init(&b, none, none)
assert(unicode_script(&a, "Greek", false) == 0)
assert(unicode_script(&b, "Grek", false) == 0)
assert(range_hash(&a) == range_hash(&b) and a.len > 0)
a.len = 0
b.len = 0
assert(unicode_general_category(&a, "Nd") == 0)
assert(unicode_general_category(&b, "digit") == 0)
assert(range_is(&a, cr_points(&b)))
a.len = 0
assert(unicode_prop(&a, "space") == 0)
assert(range_is(&a, [9, 14, 0x20, 0x21, 0x85, 0x86, 0xa0, 0xa1, 0x1680, 0x1681, 0x2000,
0x200b, 0x2028, 0x202a, 0x202f, 0x2030, 0x205f, 0x2060, 0x3000, 0x3001]))
assert(unicode_script(&a, "Klingon", false) == -2)
assert(unicode_general_category(&a, "Lx") == -2)
assert(unicode_prop(&a, "Cased1") == -2)
assert(unicode_prop(&a, "") == -2)
cr_free(&a)
cr_free(&b)
# The state of a sequence property callback: how many sequences and their hash.
struct SequenceStats:
var count: u32
var h: u32
func sequence_counter(opaque: void*?, sequence: const u32[]):
let stats = (SequenceStats*)(opaque else trap("no stats"))
stats.count += 1
stats.h = mix(stats.h, (u32)sequence.length)
for v in sequence:
stats.h = mix(stats.h, v)
test "sequence properties":
let rows = expect_sequence_properties
let n = name_count(unicode_sequence_prop_name_table)
assert((usize)n * 3 == rows.length)
for j: i32 in 0..<n:
let name = first_alias(unicode_sequence_prop_name_table, j)
var stats = SequenceStats(count = 0, h = fnv_offset)
var cr: CharRange
cr_init(&cr, none, none)
let ret = unicode_sequence_prop(name, sequence_counter, &stats, &cr)
let at = (usize)j * 3
assert(rows[at] == (u32)ret, name)
assert(rows[at + 1] == stats.count, name)
assert(rows[at + 2] == stats.h, name)
cr_free(&cr)
var cr: CharRange
cr_init(&cr, none, none)
var stats = SequenceStats(count = 0, h = fnv_offset)
assert(unicode_sequence_prop("Emoji", sequence_counter, &stats, &cr) == -2)
cr_free(&cr)