dymokomi/luce-regex
Source of release 0.6.1
0.6.1 / src / unicode / module.lucb
#==============================================================================================
#
# module - Unicode utilities: the public interface (libunicode.h)
#
# DESCRIPTION:
# Port of QuickJS libunicode.h (lines 24-196): the character range type and its inline
# operations, the normalization forms, the code point category bits and the inline
# byte and code point predicates. The module is the port of libunicode.c; its tables
# (libunicode-table.h) are the generated `*_table.lucb` fragments.
#
# Unicode utilities
#
# Copyright (c) 2017-2018 Fabrice Bellard
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify,
# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to the following
# conditions: the above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS
# IS", WITHOUT WARRANTY OF ANY KIND.
#
#==============================================================================================
# mark: Versions and limits ====================================================================
# unicode standard version
## The major version of the Unicode standard the tables implement.
pub let libunicode_unicode_version_major: i32 = 17
## The minor version of the Unicode standard the tables implement.
pub let libunicode_unicode_version_minor: i32 = 0
## The patch version of the Unicode standard the tables implement.
pub let libunicode_unicode_version_patch: i32 = 0
# CONFIG_ALL_UNICODE is always defined: every table is included.
## The largest number of code points `lre_case_conv` writes for one code point.
pub let lre_cc_res_len_max: i32 = 3
# mark: Character ranges =======================================================================
# char ranges
## A set of code points as sorted interval bounds: `points[2i] <= c < points[2i+1]`. The
## storage grows through `realloc_func(mem_opaque, ...)`, set by `cr_init`.
pub struct CharRange:
## in points, always even
pub var len: i32
## The capacity of `points`, in points.
pub var size: i32
## points sorted by increasing value; none until the first growth
pub var points: u32*?
## The opaque argument of `realloc_func`.
pub var mem_opaque: void*?
## The reallocation hook; `cr_init` sets it.
pub var realloc_func: ReallocFunc?
## CharRangeOpEnum: the operation of `cr_op` and `cr_op1`. It stays an `i32` because callers
## compute it (`op - POP_UNION + CR_OP_UNION`), and its first member would be the reserved
## word `union` as an enum case.
pub type CharRangeOpEnum = i32
## Union of two ranges.
pub let cr_op_union: CharRangeOpEnum = 0
## Intersection of two ranges.
pub let cr_op_inter: CharRangeOpEnum = 1
## Symmetric difference of two ranges.
pub let cr_op_xor: CharRangeOpEnum = 2
## The first range minus the second.
pub let cr_op_sub: CharRangeOpEnum = 3
## Add the bound `v` to `cr`, growing it; 0, or -1 when the storage cannot grow.
pub func cr_add_point(cr: CharRange*, v: u32) -> i32:
if cr.len >= cr.size:
if cr_realloc(cr, cr.len + 1) != 0:
return -1
let points = cr.points else trap("cr_add_point: no storage")
points[(usize)cr.len] = v
cr.len += 1
return 0
## Add the interval `[c1, c2)` to `cr` without merging; 0, or -1 when it cannot grow.
pub func cr_add_interval(cr: CharRange*, c1: u32, c2: u32) -> i32:
if (cr.len + 2) > cr.size:
if cr_realloc(cr, cr.len + 2) != 0:
return -1
let points = cr.points else trap("cr_add_interval: no storage")
points[(usize)cr.len] = c1
points[(usize)cr.len + 1] = c2
cr.len += 2
return 0
## Union the closed interval `[c1, c2]` into `cr`; 0, or -1 when it cannot grow.
pub func cr_union_interval(cr: CharRange*, c1: u32, c2: u32) -> i32:
var b_pt: u32[2]
b_pt[0] = c1
b_pt[1] = c2 +% 1
return cr_op1(cr, b_pt, cr_op_union)
## The bounds of `cr` as a span (empty when it has no storage), for `cr_op` and `cr_op1`,
## whose C point-pointer and length arguments are one span here.
pub func cr_points(cr: const CharRange*) -> u32[]:
if let points = cr.points:
return u32[](points, (usize)cr.len)
var empty: u32[]
return empty
# mark: Normalization forms ====================================================================
## UnicodeNormalizationEnum: the form `unicode_normalize` produces. Bit 0 is set for the
## decomposed forms, bit 1 for the compatibility forms; callers compute it
## (`UNICODE_NFC + is_compat * 2 + ...`), so it stays an `i32`.
pub type UnicodeNormalizationEnum = i32
## Canonical decomposition followed by canonical composition.
pub let unicode_nfc: UnicodeNormalizationEnum = 0
## Canonical decomposition.
pub let unicode_nfd: UnicodeNormalizationEnum = 1
## Compatibility decomposition followed by canonical composition.
pub let unicode_nfkc: UnicodeNormalizationEnum = 2
## Compatibility decomposition.
pub let unicode_nfkd: UnicodeNormalizationEnum = 3
## The callback of `unicode_sequence_prop`: `(opaque, sequence)`, once per sequence.
pub type UnicodeSequencePropCB = func(void*?, const u32[]) -> unit
# mark: Code point categories ==================================================================
# Code point type categories
## `lre_ctype_bits`: a space.
pub let unicode_c_space: u8 = 1 << 0
## `lre_ctype_bits`: a decimal digit.
pub let unicode_c_digit: u8 = 1 << 1
## `lre_ctype_bits`: an ASCII uppercase letter.
pub let unicode_c_upper: u8 = 1 << 2
## `lre_ctype_bits`: an ASCII lowercase letter.
pub let unicode_c_lower: u8 = 1 << 3
## `lre_ctype_bits`: the underscore.
pub let unicode_c_under: u8 = 1 << 4
## `lre_ctype_bits`: the dollar sign.
pub let unicode_c_dollar: u8 = 1 << 5
## `lre_ctype_bits`: a hexadecimal digit.
pub let unicode_c_xdigit: u8 = 1 << 6
## True if the Latin-1 byte `c` is a space.
pub func lre_is_space_byte(c: u8) -> bool:
return (lre_ctype_bits[c] & unicode_c_space) != 0
## True if the Latin-1 byte `c` may start a JavaScript identifier.
pub func lre_is_id_start_byte(c: u8) -> bool:
return (lre_ctype_bits[c] & (unicode_c_upper | unicode_c_lower | unicode_c_under | unicode_c_dollar)) != 0
## True if the Latin-1 byte `c` may continue a JavaScript identifier.
pub func lre_is_id_continue_byte(c: u8) -> bool:
return (lre_ctype_bits[c] & (unicode_c_upper | unicode_c_lower | unicode_c_under | unicode_c_dollar | unicode_c_digit)) != 0
## True if the Latin-1 byte `c` is a regexp word character (`\w`).
pub func lre_is_word_byte(c: u8) -> bool:
return (lre_ctype_bits[c] & (unicode_c_upper | unicode_c_lower | unicode_c_under | unicode_c_digit)) != 0
## True if the code point `c` is a JavaScript white space or line terminator.
pub func lre_is_space(c: u32) -> bool:
if c < 256:
return lre_is_space_byte((u8)c)
else:
return lre_is_space_non_ascii(c)
## True if the code point `c` may start a JavaScript identifier.
pub func lre_js_is_ident_first(c: u32) -> bool:
if c < 128:
return lre_is_id_start_byte((u8)c)
else:
return lre_is_id_start(c)
## True if the code point `c` may continue a JavaScript identifier.
pub func lre_js_is_ident_next(c: u32) -> bool:
if c < 128:
return lre_is_id_continue_byte((u8)c)
else:
# ZWNJ and ZWJ are accepted in identifiers
if c >= 0x200C and c <= 0x200D:
return true
return lre_is_id_continue(c)