dymokomi/luce-browser-css
Source of release 0.5.1
0.5.1 / src / css_syntax / tests_token.lucb
#==============================================================================================
#
# tests_token - Token, Number and serialization, pinned to the donor
#
# DESCRIPTION:
# The donor has no unit test for these files, so the expected values come from a local
# probe program built against the reference Ladybird (Token::to_debug_string,
# Token::to_string, Number::to_string, round_to_nearest_integer, and the
# identifier/string/URL serialization through Token::to_string). The probe itself is
# not in the repository; its outputs are the tables at the end of this fragment.
#
#==============================================================================================
# mark: Helpers =================================================================================
## `text` as a String.
func test_string(text: str) -> ak.String:
return ak.string_from_str(text)
## Whether `string` holds exactly `text`.
func string_is(string: const ak.String*, text: str) -> bool:
return ak.string_view_text(ak.string_bytes_as_string_view(string)) == text
## Asserts `string` holds `text`, printing both when it does not.
func expect_string(string: const ak.String*, text: str):
if not string_is(string, text):
print(f"expected: {text}")
print(f"actual: {ak.string_view_text(ak.string_bytes_as_string_view(string))}")
assert(false, "string mismatch")
## Tokenizes `input` and checks `to_debug_string|to_string` of every token against `expected`.
func check_token_strings(input: str, expected: const str[]):
let tokens = tokenizer_tokenize_string_view(ak.sv(input), ak.sv("utf-8"))
let span = ak.vector_span_const(&tokens)
assert(span.length == expected.length, "token count")
for index in 0..<span.length:
var builder = ak.string_builder_new()
let debug_string = token_to_debug_string(&span[index])
string_builder_append_string(&builder, &debug_string)
ak.string_builder_append_u8(&builder, '|')
let string = token_to_string(&span[index])
string_builder_append_string(&builder, &string)
let line = ak.string_builder_to_string_without_validation(&builder)
expect_string(&line, expected[index])
# mark: Tests ===================================================================================
noinline func test_token_to_string_1():
check_token_strings(to_string_input_1, to_string_expected_1)
test "Token::to_string and to_debug_string over every token type":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_token_to_string_1()
noinline func test_token_to_string_2():
check_token_strings(to_string_input_2, to_string_expected_2)
test "Token::to_string: control characters, clamped numbers, escapes":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_token_to_string_2()
noinline func test_number_serialization():
let values = number_serialization_values()
for index in 0..<values.length:
let value = values[index]
var builder = ak.string_builder_new()
let kinds: NumberType[3] = [NumberType.number, NumberType.integer, NumberType.integer_with_explicit_sign]
for kind in kinds:
let number = number_create(kind, value)
let string = number_to_string(&number)
string_builder_append_string(&builder, &string)
ak.string_builder_append_u8(&builder, '|')
let number_token = token_create_number(number_create(NumberType.number, value), ak.string_new())
let number_string = token_to_string(&number_token)
string_builder_append_string(&builder, &number_string)
ak.string_builder_append_u8(&builder, '|')
let percentage_token = token_create_percentage(number_create(NumberType.number, value), ak.string_new())
let percentage_string = token_to_string(&percentage_token)
string_builder_append_string(&builder, &percentage_string)
let line = ak.string_builder_to_string_without_validation(&builder)
expect_string(&line, number_serialization_expected[index])
let specials: Number[4] = [
number_create(NumberType.number, math.infinity),
number_create(NumberType.number, -math.infinity),
number_create(NumberType.number, math.nan),
number_create(NumberType.integer_with_explicit_sign, math.infinity),
]
for index in 0..<specials.length:
let string = number_to_string(&specials[index])
expect_string(&string, number_special_expected[index])
test "Number::to_string and serialize_a_number":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_number_serialization()
noinline func test_round_to_nearest_integer():
let values: f64[10] = [0.5, -0.5, 1.5, -1.5, 2.4999, 1e10, -1e10, math.nan, math.infinity, -math.infinity]
for index in 0..<values.length:
assert(round_to_nearest_integer(values[index]) == round_expected[index])
test "round_to_nearest_integer":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_round_to_nearest_integer()
noinline func test_number_arithmetic():
let two = number_create(NumberType.integer, 2.0)
let three = number_create(NumberType.integer_with_explicit_sign, 3.0)
let half = number_create(NumberType.number, 0.5)
let sum = number_add(&two, three)
assert(number_type(&sum) == NumberType.integer and number_value(&sum) == 5.0)
let difference = number_sub(&two, half)
assert(number_type(&difference) == NumberType.number and number_value(&difference) == 1.5)
let product = number_mul(&two, three)
assert(number_type(&product) == NumberType.integer and number_value(&product) == 6.0)
# Division is always a Number, even of two integers.
let quotient = number_div(&three, two)
assert(number_type("ient) == NumberType.number and number_value("ient) == 1.5)
assert(number_is_integer(&three) and number_is_integer_with_explicit_sign(&three))
assert(not number_is_integer(&half))
let two_and_a_half = number_create(NumberType.number, 2.5)
assert(number_integer_value(&two_and_a_half) == 3)
assert(number_eq(&two, number_create(NumberType.integer, 2.0)))
assert(not number_eq(&two, number_create(NumberType.number, 2.0)))
assert(number_cmp(&two, three) == -1 and number_cmp(&three, two) == 1 and number_cmp(&two, number_create(NumberType.number, 2.0)) == 0)
var zero = number_create(NumberType.integer, 7.0)
number_construct(&zero)
assert(number_type(&zero) == NumberType.number and number_value(&zero) == 0.0)
test "Number arithmetic, comparison and integer conversion":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_number_arithmetic()
noinline func test_identifier_serialization():
for index in 0..<serialization_names.length:
let name = ak.fly_string_from_str(serialization_names[index])
let tokens: Token[6] = [
token_create_ident(name, ak.string_new()),
token_create_string(name, ak.string_new()),
token_create_url(name, ak.string_new()),
token_create_function(name, ak.string_new()),
token_create_at_keyword(name, ak.string_new()),
token_create_hash(name, TokenHashType.id, ak.string_new()),
]
var builder = ak.string_builder_new()
for token_index in 0..<tokens.length:
if token_index > 0:
ak.string_builder_append_u8(&builder, ' ')
ak.string_builder_append_u8(&builder, '[')
let string = token_to_string(&tokens[token_index])
string_builder_append_string(&builder, &string)
ak.string_builder_append_u8(&builder, ']')
let line = ak.string_builder_to_string_without_validation(&builder)
expect_string(&line, serialization_expected[index])
# The String forms of the serializers answer what the builder forms append.
let identifier = serialize_an_identifier_string_view(ak.sv("-1a"))
expect_string(&identifier, "-\\31 a")
let string = serialize_a_string_string_view(ak.sv("a\"b"))
expect_string(&string, "\"a\\\"b\"")
let url = serialize_a_url_string_view(ak.sv("x"))
expect_string(&url, "url(\"x\")")
let number = serialize_a_number_f64(1.0 / 3.0)
expect_string(&number, "0.333333")
test "serialize an identifier, a string and a URL":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_identifier_serialization()
noinline func test_token_accessors():
let dimension = token_create_dimension_f64(1e40, ak.fly_string_from_str("px"), test_string("1e40px"))
# Token values are clamped to single precision when read.
assert(token_dimension_value(&dimension) == (f64)ak.f32_max)
assert(token_dimension_value_int(&dimension) == ak.i32_max)
assert(not token_is_integer(&dimension))
let negative = token_create_number(number_create(NumberType.integer, -1e40), ak.string_new())
assert(token_number_value(&negative) == (f64)ak.f32_lowest)
assert(token_to_integer(&negative) == ak.i32_min)
let delim = token_create_delim(0x2f, ak.string_new())
assert(token_delim(&delim) == 0x2f)
let open = token_create(TokenType.open_square, ak.string_new())
assert(token_mirror_variant(&open) == TokenType.close_square)
assert(ak.string_view_text(token_bracket_string(&open)) == "[")
assert(ak.string_view_text(token_bracket_mirror_string(&open)) == "]")
let ident = token_create_ident(ak.fly_string_from_str("a"), ak.string_new())
assert(token_mirror_variant(&ident) == TokenType.invalid)
assert(ak.string_view_text(token_bracket_string(&ident)) == "")
# operator== ignores the source text and positions.
var other = token_create_ident(ak.fly_string_from_str("a"), test_string("a"))
token_set_position_range(&other, TokenPosition(line = 1, column = 2), TokenPosition(line = 1, column = 3))
assert(token_eq(&ident, other))
let start = token_start_position(&other)
let end = token_end_position(&other)
assert(start.column == 2 and end.column == 3)
assert(not token_eq(&ident, token_create_ident(ak.fly_string_from_str("b"), ak.string_new())))
assert(not token_eq(&ident, token_create_function(ak.fly_string_from_str("a"), ak.string_new())))
let hash_token = token_create_hash(ak.fly_string_from_str("x"), TokenHashType.unrestricted, ak.string_new())
assert(token_hash_type(&hash_token) == TokenHashType.unrestricted)
let default_token = token_new()
assert(token_type(&default_token) == TokenType.invalid and token_hash_type_raw(&default_token) == TokenHashType.unrestricted)
let eof = tokenizer_create_eof_token()
assert(token_is(&eof, TokenType.end_of_file))
let eof_source = token_original_source_text(&eof)
expect_string(&eof_source, "")
## The token's hash type without Token::hash_type's VERIFY (for the default token).
func token_hash_type_raw(token: const Token*) -> TokenHashType:
return token.m_hash_type
test "Token accessors, clamping, brackets and equality":
# Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
# test allocates from the process heap.
test_token_accessors()
# mark: Tables from the donor ===================================================================
## Tokenizer input 1 of the local probe (Token::to_debug_string and Token::to_string).
let to_string_input_1: str = "a -b --c \\31 x #id #1x @media f( url(a b) url( x ) url(\"q\") 'str\"' \"x\\\\y\" 12 -3.5 +4 1e3 50% 2.5px 7EM < ! ~ . , : ; [ ] ( ) { } <!-- --> \\\n bad\"\n"
## `to_debug_string|to_string` of each token of to_string_input_1, from the donor.
let to_string_expected_1: str[78] = [
"Ident(value=\"a\", source=\"a\", start=0:0, end=0:1)|a",
"Whitespace(source=\" \", start=0:1, end=0:2)| ",
"Ident(value=\"-b\", source=\"-b\", start=0:2, end=0:4)|-b",
"Whitespace(source=\" \", start=0:4, end=0:5)| ",
"Ident(value=\"--c\", source=\"--c\", start=0:5, end=0:8)|--c",
"Whitespace(source=\" \", start=0:8, end=0:9)| ",
"Ident(value=\"1x\", source=\"\\\\31 x\", start=0:9, end=0:14)|\\31 x",
"Whitespace(source=\" \", start=0:14, end=0:15)| ",
"Hash(value=\"id\", hash_type=Id, source=\"#id\", start=0:15, end=0:18)|#id",
"Whitespace(source=\" \", start=0:18, end=0:19)| ",
"Hash(value=\"1x\", hash_type=Unrestricted, source=\"#1x\", start=0:19, end=0:22)|#1x",
"Whitespace(source=\" \", start=0:22, end=0:23)| ",
"AtKeyword(value=\"media\", source=\"@media\", start=0:23, end=0:29)|@media",
"Whitespace(source=\" \", start=0:29, end=0:30)| ",
"Function(value=\"f\", source=\"f(\", start=0:30, end=0:32)|f(",
"Whitespace(source=\" \", start=0:32, end=0:33)| ",
"BadUrl(source=\"url(a b)\", start=0:33, end=0:41)|url()",
"Whitespace(source=\" \", start=0:41, end=0:42)| ",
"Url(value=\"x\", source=\"url( x )\", start=0:42, end=0:52)|url(\"x\")",
"Whitespace(source=\" \", start=0:52, end=0:53)| ",
"Function(value=\"url\", source=\"url(\", start=0:53, end=0:57)|url(",
"String(value=\"q\", source=\"\\\"q\\\"\", start=0:57, end=0:60)|\"q\"",
"CloseParen(source=\")\", start=0:60, end=0:61)|)",
"Whitespace(source=\" \", start=0:61, end=0:62)| ",
"String(value=\"str\\\"\", source=\"'str\\\"'\", start=0:62, end=0:68)|\"str\\\"\"",
"Whitespace(source=\" \", start=0:68, end=0:69)| ",
"String(value=\"x\\\\y\", source=\"\\\"x\\\\\\\\y\\\"\", start=0:69, end=0:75)|\"x\\\\y\"",
"Whitespace(source=\" \", start=0:75, end=0:76)| ",
"Number(value=12, number_type=Integer, source=\"12\", start=0:76, end=0:78)|12",
"Whitespace(source=\" \", start=0:78, end=0:79)| ",
"Number(value=-3.5, number_type=Number, source=\"-3.5\", start=0:79, end=0:83)|-3.5",
"Whitespace(source=\" \", start=0:83, end=0:84)| ",
"Number(value=4, number_type=IntegerWithExplicitSign, source=\"+4\", start=0:84, end=0:86)|4",
"Whitespace(source=\" \", start=0:86, end=0:87)| ",
"Number(value=1000, number_type=Number, source=\"1e3\", start=0:87, end=0:90)|1000",
"Whitespace(source=\" \", start=0:90, end=0:91)| ",
"Percentage(value=50, number_type=Integer, source=\"50%\", start=0:91, end=0:94)|50%",
"Whitespace(source=\" \", start=0:94, end=0:95)| ",
"Dimension(value=2.5, number_type=Number, unit=\"px\", source=\"2.5px\", start=0:95, end=0:100)|2.5px",
"Whitespace(source=\" \", start=0:100, end=0:101)| ",
"Dimension(value=7, number_type=Integer, unit=\"EM\", source=\"7EM\", start=0:101, end=0:104)|7EM",
"Whitespace(source=\" \", start=0:104, end=0:105)| ",
"Delim(value=\"<\", code_point=U+003C, source=\"<\", start=0:105, end=0:106)|<",
"Whitespace(source=\" \", start=0:106, end=0:107)| ",
"Delim(value=\"!\", code_point=U+0021, source=\"!\", start=0:107, end=0:108)|!",
"Whitespace(source=\" \", start=0:108, end=0:109)| ",
"Delim(value=\"~\", code_point=U+007E, source=\"~\", start=0:109, end=0:110)|~",
"Whitespace(source=\" \", start=0:110, end=0:111)| ",
"Delim(value=\".\", code_point=U+002E, source=\".\", start=0:111, end=0:112)|.",
"Whitespace(source=\" \", start=0:112, end=0:113)| ",
"Comma(source=\",\", start=0:113, end=0:114)|,",
"Whitespace(source=\" \", start=0:114, end=0:115)| ",
"Colon(source=\":\", start=0:115, end=0:116)|:",
"Whitespace(source=\" \", start=0:116, end=0:117)| ",
"Semicolon(source=\";\", start=0:117, end=0:118)|;",
"Whitespace(source=\" \", start=0:118, end=0:119)| ",
"OpenSquare(source=\"[\", start=0:119, end=0:120)|[",
"Whitespace(source=\" \", start=0:120, end=0:121)| ",
"CloseSquare(source=\"]\", start=0:121, end=0:122)|]",
"Whitespace(source=\" \", start=0:122, end=0:123)| ",
"OpenParen(source=\"(\", start=0:123, end=0:124)|(",
"Whitespace(source=\" \", start=0:124, end=0:125)| ",
"CloseParen(source=\")\", start=0:125, end=0:126)|)",
"Whitespace(source=\" \", start=0:126, end=0:127)| ",
"OpenCurly(source=\"{\", start=0:127, end=0:128)|{",
"Whitespace(source=\" \", start=0:128, end=0:129)| ",
"CloseCurly(source=\"}\", start=0:129, end=0:130)|}",
"Whitespace(source=\" \", start=0:130, end=0:131)| ",
"CDO(source=\"<!--\", start=0:131, end=0:135)|<!--",
"Whitespace(source=\" \", start=0:135, end=0:136)| ",
"CDC(source=\"-->\", start=0:136, end=0:139)|-->",
"Whitespace(source=\" \", start=0:139, end=0:140)| ",
"Delim(value=\"\\\\\", code_point=U+005C, source=\"\\\\\", start=0:140, end=0:141)|\\",
"Whitespace(source=\"\\n \", start=0:141, end=1:1)| ",
"Ident(value=\"bad\", source=\"bad\", start=1:1, end=1:4)|bad",
"BadString(source=\"\\\"\", start=1:4, end=1:5)|",
"Whitespace(source=\"\\n\", start=1:5, end=2:0)| ",
"__EOF__(source=\"\", start=2:0, end=2:0)|",
]
## Tokenizer input 2 of the local probe (Token::to_debug_string and Token::to_string).
let to_string_input_2: str = "\u{1}\u{7f} u+ 0.0000001 1e40 -0 1.23456789 100000000 \\0 \\D800 \\110000 \\41 B"
## `to_debug_string|to_string` of each token of to_string_input_2, from the donor.
let to_string_expected_2: str[18] = [
"Delim(value=\"\\u0001\", code_point=U+0001, source=\"\\u0001\", start=0:0, end=0:1)|\u{1}",
"Delim(value=\"\u{7f}\", code_point=U+007F, source=\"\u{7f}\", start=0:1, end=0:2)|\u{7f}",
"Whitespace(source=\" \", start=0:2, end=0:3)| ",
"Ident(value=\"u\", source=\"u\", start=0:3, end=0:4)|u",
"Delim(value=\"+\", code_point=U+002B, source=\"+\", start=0:4, end=0:5)|+",
"Whitespace(source=\" \", start=0:5, end=0:6)| ",
"Number(value=1e-7, number_type=Number, source=\"0.0000001\", start=0:6, end=0:15)|1e-7",
"Whitespace(source=\" \", start=0:15, end=0:16)| ",
"Number(value=3.4028234663852886e+38, number_type=Number, source=\"1e40\", start=0:16, end=0:20)|1e+40",
"Whitespace(source=\" \", start=0:20, end=0:21)| ",
"Number(value=0, number_type=IntegerWithExplicitSign, source=\"-0\", start=0:21, end=0:23)|0",
"Whitespace(source=\" \", start=0:23, end=0:24)| ",
"Number(value=1.23456789, number_type=Number, source=\"1.23456789\", start=0:24, end=0:34)|1.23456789",
"Whitespace(source=\" \", start=0:34, end=0:35)| ",
"Number(value=100000000, number_type=Integer, source=\"100000000\", start=0:35, end=0:44)|100000000",
"Whitespace(source=\" \", start=0:44, end=0:45)| ",
"Ident(value=\"���AB\", source=\"\\\\0 \\\\D800 \\\\110000 \\\\41 B\", start=0:45, end=0:67)|���AB",
"__EOF__(source=\"\", start=0:67, end=0:67)|",
]
## The values of the Number serialization table.
func number_serialization_values() -> f64[15]:
return [0.0, 1.0, -1.0, 1.5, 0.1, 1.0 / 3.0, 123456789.0, 1e20, 1e-7, 4e-7, 5e-7, 0.0000005, -2.5, 3.14159265, 1e300]
## `Number(Number).to_string()|Number(Integer)|Number(IntegerWithExplicitSign)|Token number to_string|Token percentage to_string`
## for each value, from the donor.
let number_serialization_expected: str[15] = [
"0|0|+0|0|0%",
"1|1|+1|1|1%",
"-1|-1|-1|-1|-1%",
"1.5|1.5|+1.5|1.5|1.5%",
"0.1|0.1|+0.1|0.1|0.1%",
"0.333333|0.333333|+0.3333333333333333|0.3333333333333333|0.3333333333333333%",
"123456789|123456789|+123456789|123456789|123456789%",
"18446744073709551615.555556|18446744073709551615.555556|+100000000000000000000|100000000000000000000|100000000000000000000%",
"0|0|+1e-7|1e-7|1e-7%",
"0|0|+4e-7|4e-7|4e-7%",
"5e-7|5e-7|+5e-7|5e-7|5e-7%",
"5e-7|5e-7|+5e-7|5e-7|5e-7%",
"-2.5|-2.5|-2.5|-2.5|-2.5%",
"3.141593|3.141593|+3.14159265|3.14159265|3.14159265%",
"1e+300|1e+300|+1e+300|1e+300|1e+300%",
]
## infinity, -infinity, NaN as Number, and +infinity as IntegerWithExplicitSign, from the donor.
let number_special_expected: str[4] = ["infinity", "-infinity", "NaN", "+inf"]
## round_to_nearest_integer of 0.5, -0.5, 1.5, -1.5, 2.4999, 1e10, -1e10, NaN, inf, -inf, from the donor.
let round_expected: i32[10] = [1, 0, 2, -1, 2, 2147483647, -2147483648, 0, 2147483647, -2147483648]
## The names of the identifier/string/url serialization table.
let serialization_names: str[11] = ["", "-", "--", "-1a", "1a", "a b", "é_x", "\u{1}z", "a\"b\\c", "a\u{7f}", "_-9"]
## `[ident] [string] [url] [function] [at-keyword] [id hash]` Token::to_string of each name, from the donor.
let serialization_expected: str[11] = [
"[] [\"\"] [url(\"\")] [(] [@] [#]",
"[\\-] [\"-\"] [url(\"-\")] [\\-(] [@\\-] [#\\-]",
"[--] [\"--\"] [url(\"--\")] [--(] [@--] [#--]",
"[-\\31 a] [\"-1a\"] [url(\"-1a\")] [-\\31 a(] [@-\\31 a] [#-\\31 a]",
"[\\31 a] [\"1a\"] [url(\"1a\")] [\\31 a(] [@\\31 a] [#\\31 a]",
"[a\\ b] [\"a b\"] [url(\"a b\")] [a\\ b(] [@a\\ b] [#a\\ b]",
"[é_x] [\"é_x\"] [url(\"é_x\")] [é_x(] [@é_x] [#é_x]",
"[\\1 z] [\"\\1 z\"] [url(\"\\1 z\")] [\\1 z(] [@\\1 z] [#\\1 z]",
"[a\\\"b\\\\c] [\"a\\\"b\\\\c\"] [url(\"a\\\"b\\\\c\")] [a\\\"b\\\\c(] [@a\\\"b\\\\c] [#a\\\"b\\\\c]",
"[a\\7f ] [\"a\\7f \"] [url(\"a\\7f \")] [a\\7f (] [@a\\7f ] [#a\\7f ]",
"[_-9] [\"_-9\"] [url(\"_-9\")] [_-9(] [@_-9] [#_-9]",
]