dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / ak / generic_lexer_2.lucb
#==============================================================================================
#
# generic_lexer_2 - GenericLexer<char16_t> (Utf16GenericLexer) and LineTrackingLexer
#
# DESCRIPTION:
# Port of AK/GenericLexer.h (lines 41-533) for CharType = char16_t, as the
# utf16_generic_lexer_* functions over `GenericLexer[u16]` (ViewType Utf16View, the
# m_utf16_input field), and of LineTrackingLexer (AK/GenericLexer.h:492-527,
# AK/GenericLexer.cpp lines 1-41). The char16_t lexer's extra overloads taking a `char`
# or a StringView are the `_u8` and `_string_view` functions; its predicates are
# CodePointPredicate structs.
#
#==============================================================================================
# mark: Predicates ==============================================================================
## The is_any_of lambda applied to a code unit (GenericLexer<char16_t> calls it with char16_t):
## `values.contains(c)`.
pub struct CodePointIsAnyOf: CodePointPredicate:
pub var values: StringView
pub func matches(code_point: u32) -> bool:
if code_point <= 0x7f:
return string_view_contains_u8(&self.values, (u8)code_point)
return string_view_contains_u32(&self.values, code_point)
## A plain function (such as is_ascii_digit) used as a CodePointPredicate.
pub struct CodePointPredicateFunction: CodePointPredicate:
pub var function: func(u32) -> bool
pub func matches(code_point: u32) -> bool:
return self.function(code_point)
# mark: Construction and position ===============================================================
## Ported from AK/GenericLexer.h:47 AK::Detail::GenericLexer::GenericLexer<char16_t>(Utf16View).
pub func utf16_generic_lexer_construct(this: GenericLexer[u16]*, input: Utf16View):
this.m_utf16_input = input
this.m_index = 0
## `Utf16GenericLexer lexer(input)`: the value form of the constructor.
pub func utf16_generic_lexer_new(input: Utf16View) -> GenericLexer[u16]:
var lexer: GenericLexer[u16]
utf16_generic_lexer_construct(&lexer, input)
return lexer
## Ported from AK/GenericLexer.h:52 AK::Detail::GenericLexer::tell.
pub func utf16_generic_lexer_tell(this: const GenericLexer[u16]*) -> usize:
return this.m_index
## Ported from AK/GenericLexer.h:53 AK::Detail::GenericLexer::tell_remaining.
pub func utf16_generic_lexer_tell_remaining(this: const GenericLexer[u16]*) -> usize:
return utf16_generic_lexer_input_length(this) - this.m_index
## Ported from AK/GenericLexer.h:55 AK::Detail::GenericLexer::remaining.
pub func utf16_generic_lexer_remaining(this: const GenericLexer[u16]*) -> Utf16View:
return utf16_view_substring_view(&this.m_utf16_input, this.m_index)
## Ported from AK/GenericLexer.h:56 AK::Detail::GenericLexer::input.
pub func utf16_generic_lexer_input(this: const GenericLexer[u16]*) -> Utf16View:
return this.m_utf16_input
## Ported from AK/GenericLexer.h:58 AK::Detail::GenericLexer::is_eof.
pub func utf16_generic_lexer_is_eof(this: const GenericLexer[u16]*) -> bool:
return this.m_index >= utf16_generic_lexer_input_length(this)
## Ported from AK/GenericLexer.h:60 AK::Detail::GenericLexer::peek.
pub func utf16_generic_lexer_peek(this: const GenericLexer[u16]*, offset: usize = 0) -> u16:
return utf16_generic_lexer_code_unit_at(this, this.m_index + offset) if this.m_index + offset < utf16_generic_lexer_input_length(this) else 0
## Ported from AK/GenericLexer.h:65 AK::Detail::GenericLexer::peek_string.
pub func utf16_generic_lexer_peek_string(this: const GenericLexer[u16]*, length: usize, offset: usize = 0) -> Utf16View?:
if this.m_index + offset + length > utf16_generic_lexer_input_length(this):
return none
return utf16_view_substring_view_usize(&this.m_utf16_input, this.m_index + offset, length)
# mark: Matching ================================================================================
## Ported from AK/GenericLexer.h:72 AK::Detail::GenericLexer::next_is(CharType).
pub func utf16_generic_lexer_next_is_t_char_type(this: const GenericLexer[u16]*, expected: u16) -> bool:
return utf16_generic_lexer_peek(this) == expected
## Ported from AK/GenericLexer.h:77 AK::Detail::GenericLexer::next_is(char).
pub func utf16_generic_lexer_next_is_u8(this: const GenericLexer[u16]*, expected: u8) -> bool:
return utf16_generic_lexer_peek(this) == expected
## Ported from AK/GenericLexer.h:83 AK::Detail::GenericLexer::next_is(ViewType).
pub func utf16_generic_lexer_next_is_utf16_view(this: const GenericLexer[u16]*, expected: Utf16View) -> bool:
let length = utf16_view_length_in_code_units(&expected)
let next = utf16_generic_lexer_peek_string(this, length) else return false
return utf16_view_eq_utf16_view(&next, &expected)
## Ported from AK/GenericLexer.h:95 AK::Detail::GenericLexer::next_is(StringView).
pub func utf16_generic_lexer_next_is_string_view(this: const GenericLexer[u16]*, expected: StringView) -> bool:
let next = utf16_generic_lexer_peek_string(this, string_view_length(&expected)) else return false
return utf16_view_eq_string_view(&next, expected)
## Ported from AK/GenericLexer.h:101 AK::Detail::GenericLexer::retreat.
pub func utf16_generic_lexer_retreat(this: GenericLexer[u16]*):
assert(this.m_index > 0)
this.m_index -= 1
## Ported from AK/GenericLexer.h:107 AK::Detail::GenericLexer::retreat(size_t).
pub func utf16_generic_lexer_retreat_usize(this: GenericLexer[u16]*, count: usize):
assert(this.m_index >= count)
this.m_index -= count
## Ported from AK/GenericLexer.h:113 AK::Detail::GenericLexer::consume.
pub func utf16_generic_lexer_consume(this: GenericLexer[u16]*) -> u16:
assert(not utf16_generic_lexer_is_eof(this))
let code_unit = utf16_generic_lexer_code_unit_at(this, this.m_index)
this.m_index += 1
return code_unit
## Ported from AK/GenericLexer.h:119 AK::Detail::GenericLexer::consume_specific(CharType).
pub func utf16_generic_lexer_consume_specific_t_char_type(this: GenericLexer[u16]*, next: u16) -> bool:
if not utf16_generic_lexer_next_is_t_char_type(this, next):
return false
utf16_generic_lexer_ignore(this)
return true
## Ported from AK/GenericLexer.h:128 AK::Detail::GenericLexer::consume_specific(char).
pub func utf16_generic_lexer_consume_specific_u8(this: GenericLexer[u16]*, next: u8) -> bool:
return utf16_generic_lexer_consume_specific_t_char_type(this, next)
## Ported from AK/GenericLexer.h:134 AK::Detail::GenericLexer::consume_specific(ViewType).
pub func utf16_generic_lexer_consume_specific_utf16_view(this: GenericLexer[u16]*, next: Utf16View) -> bool:
if not utf16_generic_lexer_next_is_utf16_view(this, next):
return false
utf16_generic_lexer_ignore(this, utf16_view_length_in_code_units(&next))
return true
## Ported from AK/GenericLexer.h:147 AK::Detail::GenericLexer::consume_specific(StringView).
pub func utf16_generic_lexer_consume_specific_string_view(this: GenericLexer[u16]*, next: StringView) -> bool:
if not utf16_generic_lexer_next_is_string_view(this, next):
return false
utf16_generic_lexer_ignore(this, string_view_length(&next))
return true
## Ported from AK/GenericLexer.h:157 AK::Detail::GenericLexer::consume_escaped_character. An
## `escape_map` without characters stands for the C++ default "n\nr\rt\tb\bf\f".
pub func utf16_generic_lexer_consume_escaped_character(this: GenericLexer[u16]*, escape_char: u16 = 0x5c, escape_map: StringView = StringView()) -> u16:
let map = sv("n\nr\rt\tb\u{8}f\u{c}") if string_view_characters_without_null_termination(&escape_map) == none else escape_map
if not utf16_generic_lexer_consume_specific_t_char_type(this, escape_char):
return utf16_generic_lexer_consume(this)
let c_ = utf16_generic_lexer_consume(this)
var i: usize = 0
while i < string_view_length(&map):
if c_ == (u16)string_view_at(&map, i):
return string_view_at(&map, i + 1)
i += 2
return c_
## Ported from AK/GenericLexer.h:173 AK::Detail::GenericLexer::consume(size_t).
pub func utf16_generic_lexer_consume_usize(this: GenericLexer[u16]*, count: usize) -> Utf16View:
let start = this.m_index
let available = utf16_generic_lexer_input_length(this) - this.m_index
let length = count if count < available else available
this.m_index += length
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:183 AK::Detail::GenericLexer::consume_all.
pub func utf16_generic_lexer_consume_all(this: GenericLexer[u16]*) -> Utf16View:
let rest = utf16_view_substring_view_usize(&this.m_utf16_input, this.m_index, utf16_generic_lexer_input_length(this) - this.m_index)
this.m_index = utf16_generic_lexer_input_length(this)
return rest
## Ported from AK/GenericLexer.h:191 AK::Detail::GenericLexer::consume_line.
pub func utf16_generic_lexer_consume_line(this: GenericLexer[u16]*) -> Utf16View:
let start = this.m_index
while not utf16_generic_lexer_is_eof(this) and utf16_generic_lexer_peek(this) != (u16)'\r' and utf16_generic_lexer_peek(this) != (u16)'\n':
this.m_index += 1
let length = this.m_index - start
_ = utf16_generic_lexer_consume_specific_u8(this, '\r')
_ = utf16_generic_lexer_consume_specific_u8(this, '\n')
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:205 AK::Detail::GenericLexer::consume_until(CharType).
pub func utf16_generic_lexer_consume_until_t_char_type(this: GenericLexer[u16]*, stop: u16) -> Utf16View:
let start = this.m_index
while not utf16_generic_lexer_is_eof(this) and utf16_generic_lexer_peek(this) != stop:
this.m_index += 1
let length = this.m_index - start
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:215 AK::Detail::GenericLexer::consume_until(char).
pub func utf16_generic_lexer_consume_until_u8(this: GenericLexer[u16]*, stop: u8) -> Utf16View:
return utf16_generic_lexer_consume_until_t_char_type(this, stop)
## Ported from AK/GenericLexer.h:222 AK::Detail::GenericLexer::consume_until(ViewType).
pub func utf16_generic_lexer_consume_until_utf16_view(this: GenericLexer[u16]*, stop: Utf16View) -> Utf16View:
let start = this.m_index
while not utf16_generic_lexer_is_eof(this) and not utf16_generic_lexer_next_is_utf16_view(this, stop):
this.m_index += 1
let length = this.m_index - start
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:235 AK::Detail::GenericLexer::consume_quoted_string.
pub func utf16_generic_lexer_consume_quoted_string(this: GenericLexer[u16]*, escape_char: u16 = 0) -> Utf16View:
let quote = CodePointIsAnyOf(values = sv("'\""))
if not utf16_generic_lexer_next_is_tt_predicate(this, "e):
return utf16_view_new()
let quote_char = utf16_generic_lexer_consume(this)
let start = this.m_index
while not utf16_generic_lexer_is_eof(this):
if utf16_generic_lexer_next_is_t_char_type(this, escape_char):
this.m_index += 1
elif utf16_generic_lexer_next_is_t_char_type(this, quote_char):
break
this.m_index += 1
let length = this.m_index - start
if utf16_generic_lexer_peek(this) != quote_char:
# Restore the index in case the string is unterminated
this.m_index = start - 1
return utf16_view_new()
# Ignore closing quote
utf16_generic_lexer_ignore(this)
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
# mark: Integers and escapes ====================================================================
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<T> (see
## generic_lexer_consume_decimal_integer_magnitude).
func utf16_generic_lexer_consume_decimal_integer_magnitude(this: GenericLexer[u16]*, maximum: u64, unsigned_maximum: u64, is_unsigned: bool) -> (u64, bool)!:
let rollback_position = this.m_index
var has_minus_sign = false
if utf16_generic_lexer_next_is_u8(this, '+') or utf16_generic_lexer_next_is_u8(this, '-'):
if utf16_generic_lexer_consume(this) == (u16)'-':
has_minus_sign = true
let digits = CodePointPredicateFunction(function = is_ascii_digit)
let number_view = utf16_generic_lexer_consume_while(this, &digits)
if utf16_view_is_empty(&number_view):
this.m_index = rollback_position
error(ak_errno_error, "EINVAL")
let parsed = utf16_view_to_number_u64(&number_view, TrimWhitespace.no)
let number = parsed else 0
if parsed == none or number > unsigned_maximum:
this.m_index = rollback_position
error(ak_errno_error, "ERANGE")
return try consume_decimal_integer_check_range(&this.m_index, rollback_position, number, has_minus_sign, maximum, is_unsigned)
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<i8>.
pub func utf16_generic_lexer_consume_decimal_integer_i8(this: GenericLexer[u16]*) -> i8!:
let (magnitude, negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0x7f, 0xff, false)
return (i8)(0 -% magnitude) if negative else (i8)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<i16>.
pub func utf16_generic_lexer_consume_decimal_integer_i16(this: GenericLexer[u16]*) -> i16!:
let (magnitude, negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0x7fff, 0xffff, false)
return (i16)(0 -% magnitude) if negative else (i16)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<i32>.
pub func utf16_generic_lexer_consume_decimal_integer_i32(this: GenericLexer[u16]*) -> i32!:
let (magnitude, negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0x7fffffff, 0xffffffff, false)
return (i32)(0 -% magnitude) if negative else (i32)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<i64>.
pub func utf16_generic_lexer_consume_decimal_integer_i64(this: GenericLexer[u16]*) -> i64!:
let (magnitude, negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0x7fffffffffffffff, 0xffffffffffffffff, false)
return (i64)(0 -% magnitude) if negative else (i64)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<u8>.
pub func utf16_generic_lexer_consume_decimal_integer_u8(this: GenericLexer[u16]*) -> u8!:
let (magnitude, _negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0xff, 0xff, true)
return (u8)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<u16>.
pub func utf16_generic_lexer_consume_decimal_integer_u16(this: GenericLexer[u16]*) -> u16!:
let (magnitude, _negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0xffff, 0xffff, true)
return (u16)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<u32>.
pub func utf16_generic_lexer_consume_decimal_integer_u32(this: GenericLexer[u16]*) -> u32!:
let (magnitude, _negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0xffffffff, 0xffffffff, true)
return (u32)magnitude
## Ported from AK/GenericLexer.h:264 AK::Detail::GenericLexer::consume_decimal_integer<u64>.
pub func utf16_generic_lexer_consume_decimal_integer_u64(this: GenericLexer[u16]*) -> u64!:
let (magnitude, _negative) = try utf16_generic_lexer_consume_decimal_integer_magnitude(this, 0xffffffffffffffff, 0xffffffffffffffff, true)
return magnitude
## Ported from AK/GenericLexer.h:311 AK::Detail::GenericLexer::consume_escaped_code_point.
pub func utf16_generic_lexer_consume_escaped_code_point(this: GenericLexer[u16]*, combine_surrogate_pairs: bool = true) -> UnicodeEscapeResult:
if not utf16_generic_lexer_consume_specific_string_view(this, sv("\\u")):
return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.malformed_unicode_escape)
if utf16_generic_lexer_next_is_u8(this, '{'):
return utf16_generic_lexer_decode_code_point(this)
return utf16_generic_lexer_decode_single_or_paired_surrogate(this, combine_surrogate_pairs)
## Ported from AK/GenericLexer.h:321 AK::Detail::GenericLexer::ignore.
pub func utf16_generic_lexer_ignore(this: GenericLexer[u16]*, count: usize = 1):
let available = utf16_generic_lexer_input_length(this) - this.m_index
this.m_index += count if count < available else available
## Ported from AK/GenericLexer.h:327 AK::Detail::GenericLexer::ignore_until(CharType).
pub func utf16_generic_lexer_ignore_until_t_char_type(this: GenericLexer[u16]*, stop: u16):
while not utf16_generic_lexer_is_eof(this) and utf16_generic_lexer_peek(this) != stop:
this.m_index += 1
## Ported from AK/GenericLexer.h:333 AK::Detail::GenericLexer::ignore_until(char).
pub func utf16_generic_lexer_ignore_until_u8(this: GenericLexer[u16]*, stop: u8):
utf16_generic_lexer_ignore_until_t_char_type(this, stop)
# mark: Predicates ==============================================================================
## Ported from AK/GenericLexer.h:349 AK::Detail::GenericLexer::next_is(TPredicate).
pub func utf16_generic_lexer_next_is_tt_predicate[TTPredicate: CodePointPredicate](this: const GenericLexer[u16]*, pred: const TTPredicate*) -> bool:
return pred.matches(utf16_generic_lexer_peek(this))
## Ported from AK/GenericLexer.h:356 AK::Detail::GenericLexer::consume_while.
pub func utf16_generic_lexer_consume_while[TTPredicate: CodePointPredicate](this: GenericLexer[u16]*, pred: const TTPredicate*) -> Utf16View:
let start = this.m_index
while not utf16_generic_lexer_is_eof(this) and pred.matches(utf16_generic_lexer_peek(this)):
this.m_index += 1
let length = this.m_index - start
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:368 AK::Detail::GenericLexer::consume_until(TPredicate).
pub func utf16_generic_lexer_consume_until_tt_predicate[TTPredicate: CodePointPredicate](this: GenericLexer[u16]*, pred: const TTPredicate*) -> Utf16View:
let start = this.m_index
while not utf16_generic_lexer_is_eof(this) and not pred.matches(utf16_generic_lexer_peek(this)):
this.m_index += 1
let length = this.m_index - start
return utf16_view_substring_view_usize(&this.m_utf16_input, start, length)
## Ported from AK/GenericLexer.h:379 AK::Detail::GenericLexer::consume_specific_with_predicate.
pub func utf16_generic_lexer_consume_specific_with_predicate[TTPredicate: CodePointPredicate](this: GenericLexer[u16]*, pred: const TTPredicate*) -> bool:
if utf16_generic_lexer_is_eof(this) or not pred.matches(utf16_generic_lexer_peek(this)):
return false
utf16_generic_lexer_ignore(this)
return true
## Ported from AK/GenericLexer.h:390 AK::Detail::GenericLexer::ignore_while.
pub func utf16_generic_lexer_ignore_while[TTPredicate: CodePointPredicate](this: GenericLexer[u16]*, pred: const TTPredicate*):
while not utf16_generic_lexer_is_eof(this) and pred.matches(utf16_generic_lexer_peek(this)):
this.m_index += 1
## Ported from AK/GenericLexer.h:398 AK::Detail::GenericLexer::ignore_until(TPredicate).
pub func utf16_generic_lexer_ignore_until_tt_predicate[TTPredicate: CodePointPredicate](this: GenericLexer[u16]*, pred: const TTPredicate*):
while not utf16_generic_lexer_is_eof(this) and not pred.matches(utf16_generic_lexer_peek(this)):
this.m_index += 1
# mark: Decoding ================================================================================
## Ported from AK/GenericLexer.h:405 AK::Detail::GenericLexer::decode_code_point.
pub func utf16_generic_lexer_decode_code_point(this: GenericLexer[u16]*) -> UnicodeEscapeResult:
let starts_with_open_bracket = utf16_generic_lexer_consume_specific_u8(this, '{')
assert(starts_with_open_bracket)
let hex_digit = CodePointPredicateFunction(function = is_ascii_hex_digit)
var code_point: u32 = 0
while true:
if not utf16_generic_lexer_next_is_tt_predicate(this, &hex_digit):
return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.malformed_unicode_escape)
let new_code_point = (code_point << 4) | parse_ascii_hex_digit(utf16_generic_lexer_consume(this))
if new_code_point < code_point:
return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.unicode_escape_overflow)
code_point = new_code_point
if utf16_generic_lexer_consume_specific_u8(this, '}'):
break
if is_unicode(code_point):
return UnicodeEscapeResult.success(code_point = code_point)
return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.unicode_escape_overflow)
## The `decode_one_surrogate` lambda of decode_single_or_paired_surrogate.
func utf16_generic_lexer_decode_one_surrogate(this: GenericLexer[u16]*) -> u16?:
let surrogate_length: usize = 4
let hex_digit = CodePointPredicateFunction(function = is_ascii_hex_digit)
var surrogate: u16 = 0
for _i in 0..<surrogate_length:
if not utf16_generic_lexer_next_is_tt_predicate(this, &hex_digit):
return none
surrogate = (surrogate << 4) | (u16)parse_ascii_hex_digit(utf16_generic_lexer_consume(this))
return surrogate
## Ported from AK/GenericLexer.h:430 AK::Detail::GenericLexer::decode_single_or_paired_surrogate.
pub func utf16_generic_lexer_decode_single_or_paired_surrogate(this: GenericLexer[u16]*, combine_surrogate_pairs: bool = true) -> UnicodeEscapeResult:
let high_surrogate = utf16_generic_lexer_decode_one_surrogate(this) else return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.malformed_unicode_escape)
if not unicode_utils_is_utf16_high_surrogate(high_surrogate):
return UnicodeEscapeResult.success(code_point = high_surrogate)
if not combine_surrogate_pairs or not utf16_generic_lexer_consume_specific_string_view(this, sv("\\u")):
return UnicodeEscapeResult.success(code_point = high_surrogate)
if utf16_generic_lexer_next_is_u8(this, '{'):
utf16_generic_lexer_retreat_usize(this, 2)
return UnicodeEscapeResult.success(code_point = high_surrogate)
let low_surrogate = utf16_generic_lexer_decode_one_surrogate(this) else return UnicodeEscapeResult.failure(reason = UnicodeEscapeError.malformed_unicode_escape)
if unicode_utils_is_utf16_low_surrogate(low_surrogate):
return UnicodeEscapeResult.success(code_point = unicode_utils_decode_utf16_surrogate_pair(high_surrogate, low_surrogate))
utf16_generic_lexer_retreat_usize(this, 6)
return UnicodeEscapeResult.success(code_point = high_surrogate)
## Ported from AK/GenericLexer.h:470 AK::Detail::GenericLexer::input_length.
pub func utf16_generic_lexer_input_length(this: const GenericLexer[u16]*) -> usize:
return utf16_view_length_in_code_units(&this.m_utf16_input)
## Ported from AK/GenericLexer.h:478 AK::Detail::GenericLexer::code_unit_at.
pub func utf16_generic_lexer_code_unit_at(this: const GenericLexer[u16]*, index: usize) -> u16:
return utf16_view_code_unit_at(&this.m_utf16_input, index)
# mark: LineTrackingLexer =======================================================================
## Ported from AK/GenericLexer.h:500 AK::LineTrackingLexer::LineTrackingLexer(StringView, Position).
pub func line_tracking_lexer_construct_line_tracking_lexer_position(this: LineTrackingLexer*, input: StringView, start_position: LineTrackingLexerPosition):
generic_lexer_construct(&this.generic_lexer, input)
this.m_first_line_start_position = start_position
this.m_line_start_positions = new RedBlackTree[usize, usize] else trap("out of memory")
this.m_largest_known_line_start_position = 0
red_black_tree_insert(this.m_line_start_positions, 0, 0)
var first_newline = string_view_length(&input)
if let newline = string_view_find_u8(&input, '\n'):
first_newline = newline + 1
red_black_tree_insert(this.m_line_start_positions, first_newline, 1)
this.m_largest_known_line_start_position = first_newline
## Ported from AK/GenericLexer.h:511 AK::LineTrackingLexer::LineTrackingLexer(StringView).
pub func line_tracking_lexer_construct(this: LineTrackingLexer*, input: StringView):
line_tracking_lexer_construct_line_tracking_lexer_position(this, input, LineTrackingLexerPosition(offset = 0, line = 1, column = 1))
## `m_line_start_positions->size()`: a RedBlackTree starts with its BaseRedBlackTree.
func line_tracking_lexer_known_line_count(this: const LineTrackingLexer*) -> usize:
return base_red_black_tree_size((const BaseRedBlackTree[usize]*)this.m_line_start_positions)
## Ported from AK/GenericLexer.cpp:11 AK::LineTrackingLexer::position_for: the line and column
## of `index` (line starts are cached in the `mutable` tree).
pub func line_tracking_lexer_position_for(this: const LineTrackingLexer*, index: usize) -> LineTrackingLexerPosition:
let mutable_this = (LineTrackingLexer*)this
let input = &this.generic_lexer.m_input
# Sad case: we have no idea where the nearest newline is, so we have to scan ahead a bit.
while index > this.m_largest_known_line_start_position:
let maybe_next_newline = string_view_find_u8(input, '\n', this.m_largest_known_line_start_position)
if maybe_next_newline == none:
# No more newlines, add the end of the input as a line start to avoid searching again.
red_black_tree_insert(mutable_this.m_line_start_positions, generic_lexer_input_length(&this.generic_lexer), line_tracking_lexer_known_line_count(this))
mutable_this.m_largest_known_line_start_position = generic_lexer_input_length(&this.generic_lexer)
break
let next_newline = maybe_next_newline else trap("unreachable")
red_black_tree_insert(mutable_this.m_line_start_positions, next_newline + 1, line_tracking_lexer_known_line_count(this))
mutable_this.m_largest_known_line_start_position = next_newline + 1
# We should always have at least the first line start position.
var previous_line_it = red_black_tree_find_largest_not_above_iterator(mutable_this.m_line_start_positions, index)
let previous_line_index = red_black_tree_iterator_key(&previous_line_it)
var line = *red_black_tree_iterator_deref(&previous_line_it)
var column = index - previous_line_index
if line == 0:
# First line, take into account the start position.
column += this.m_first_line_start_position.column
line += this.m_first_line_start_position.line
return LineTrackingLexerPosition(offset = index, line = line, column = column)
## Ported from AK/GenericLexer.h:517 AK::LineTrackingLexer::current_position.
pub func line_tracking_lexer_current_position(this: const LineTrackingLexer*) -> LineTrackingLexerPosition:
return line_tracking_lexer_position_for(this, this.generic_lexer.m_index)