Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-html

Source of release 0.5.1

0.5.1 / src / html_syntax / tests_entities.lucb

#==============================================================================================
#
#   tests_entities - the named character reference matcher and its tables
#
#   DESCRIPTION:
#       The donor has no unit test for Entities.cpp: these tests match every entry of
#       Entities.json (tests_entities_table) through NamedCharacterReferenceMatcher, and pin
#       the matcher's state after each code point to the donor's (a local oracle linking
#       Entities.cpp and the generated NamedCharacterReferences.cpp of the reference build,
#       .donors/luce-browser-tools/html_tokenizer_oracle, `oracle --matcher WORD`).
#
#==============================================================================================

# mark: Every named character reference =========================================================

test "every named character reference of Entities.json":
    for entity in entity_cases:
        var matcher = NamedCharacterReferenceMatcher()
        for byte in entity.name.bytes:
            assert(named_character_reference_matcher_try_consume_ascii_char(&matcher, byte))
        let codepoints = named_character_reference_matcher_code_points(&matcher) else trap("no match")
        assert(codepoints.first == entity.first)
        assert((u32)(named_character_reference_second_codepoint_value(codepoints.second) else 0) == entity.second)
        assert(named_character_reference_matcher_overconsumed_code_points(&matcher) == 0)
        let ends_with_semicolon = entity.name.bytes[entity.name.length - 1] == ';'
        assert(named_character_reference_matcher_last_match_ends_with_semicolon(&matcher) == ends_with_semicolon)

# mark: The tables ==============================================================================

test "named_character_reference_second_codepoint_value":
    assert(named_character_reference_second_codepoint_value(.none_) == none)
    assert(named_character_reference_second_codepoint_value(.combining_long_solidus_overlay) == 0x0338)
    assert(named_character_reference_second_codepoint_value(.combining_long_vertical_line_overlay) == 0x20D2)
    assert(named_character_reference_second_codepoint_value(.hair_space) == 0x200A)
    assert(named_character_reference_second_codepoint_value(.combining_double_low_line) == 0x0333)
    assert(named_character_reference_second_codepoint_value(.combining_reverse_solidus_overlay) == 0x20E5)
    assert(named_character_reference_second_codepoint_value(.variation_selector1) == 0xFE00)
    assert(named_character_reference_second_codepoint_value(.latin_small_letter_j) == 0x006A)
    assert(named_character_reference_second_codepoint_value(.combining_macron_below) == 0x0331)

test "named_character_reference_codepoints_from_unique_index":
    # Note: The unique index is 1-based.
    assert(named_character_reference_codepoints_from_unique_index(0) == none)
    let first = named_character_reference_codepoints_from_unique_index(1) else trap("none")
    assert(first.first == 0xC6 and first.second == NamedCharacterReferenceSecondCodepoint.none_)
    let last = named_character_reference_codepoints_from_unique_index(2231) else trap("none")
    assert(last.first == 0x200C and last.second == NamedCharacterReferenceSecondCodepoint.none_)
    assert(g_named_character_reference_first_layer.length == 52)
    assert(g_named_character_reference_first_to_second_layer.length == 52)
    assert(g_named_character_reference_second_layer.length == 630)
    assert(g_named_character_reference_nodes.length == 3190)

# mark: The matcher step by step ================================================================

# Lowercase hexadecimal without leading zeros, AK's "{:x}".
func append_hex(builder: ak.StringBuilder*, value: u32):
    let digits = "0123456789abcdef".bytes
    var text: u8[8]
    var count: usize = 0
    var rest = value
    while true:
        text[count] = digits[rest & 0xF]
        rest = rest >> 4
        count += 1
        if rest == 0:
            break
    while count > 0:
        count -= 1
        ak.string_builder_append_u8(builder, text[count])

# The oracle's `--matcher WORD`: one line per code point of `word`.
func check_matcher(word: str, expected: str):
    var matcher = NamedCharacterReferenceMatcher()
    var builder = ak.string_builder_new()
    var buffer: u8[96]
    for character in word:
        let c_ = (u32)character
        let consumed = named_character_reference_matcher_try_consume_code_point(&matcher, c_)
        append_hex(&builder, c_)
        ak.string_builder_append(&builder, strings.format(buffer, f" consumed={(i32)consumed} overconsumed={named_character_reference_matcher_overconsumed_code_points(&matcher)} semicolon={(i32)named_character_reference_matcher_last_match_ends_with_semicolon(&matcher)} match=") else trap("format"))
        if let codepoints = named_character_reference_matcher_code_points(&matcher):
            append_hex(&builder, codepoints.first)
            let second = named_character_reference_second_codepoint_value(codepoints.second) else 0
            ak.string_builder_append(&builder, strings.format(buffer, f"/{second}") else trap("format"))
        else:
            ak.string_builder_append(&builder, "none")
        ak.string_builder_append(&builder, "\n")
    let actual = ak.string_builder_string_view(&builder)
    if not ak.string_view_equals(&actual, expected):
        print(f"word: {word}\nexpected:\n{expected}actual:\n{(str)ak.string_view_bytes(&actual)}")
        trap("the matcher differs from the donor's")

test "the matcher, code point by code point":
    with memory.heap:
        check_matcher("notindot;x", """
            6e consumed=1 overconsumed=1 semicolon=0 match=none
            6f consumed=1 overconsumed=2 semicolon=0 match=none
            74 consumed=1 overconsumed=0 semicolon=0 match=ac/0
            69 consumed=1 overconsumed=1 semicolon=0 match=ac/0
            6e consumed=1 overconsumed=2 semicolon=0 match=ac/0
            64 consumed=1 overconsumed=3 semicolon=0 match=ac/0
            6f consumed=1 overconsumed=4 semicolon=0 match=ac/0
            74 consumed=1 overconsumed=5 semicolon=0 match=ac/0
            3b consumed=1 overconsumed=0 semicolon=1 match=22f5/824
            78 consumed=0 overconsumed=0 semicolon=1 match=22f5/824
            """)
        check_matcher("1AĆ©&fjlig", """
            31 consumed=0 overconsumed=0 semicolon=0 match=none
            41 consumed=1 overconsumed=1 semicolon=0 match=none
            e9 consumed=0 overconsumed=1 semicolon=0 match=none
            26 consumed=0 overconsumed=1 semicolon=0 match=none
            66 consumed=1 overconsumed=2 semicolon=0 match=none
            6a consumed=0 overconsumed=2 semicolon=0 match=none
            6c consumed=0 overconsumed=2 semicolon=0 match=none
            69 consumed=0 overconsumed=2 semicolon=0 match=none
            67 consumed=0 overconsumed=2 semicolon=0 match=none
            """)
        check_matcher("NotEqualTilde;", """
            4e consumed=1 overconsumed=1 semicolon=0 match=none
            6f consumed=1 overconsumed=2 semicolon=0 match=none
            74 consumed=1 overconsumed=3 semicolon=0 match=none
            45 consumed=1 overconsumed=4 semicolon=0 match=none
            71 consumed=1 overconsumed=5 semicolon=0 match=none
            75 consumed=1 overconsumed=6 semicolon=0 match=none
            61 consumed=1 overconsumed=7 semicolon=0 match=none
            6c consumed=1 overconsumed=8 semicolon=0 match=none
            54 consumed=1 overconsumed=9 semicolon=0 match=none
            69 consumed=1 overconsumed=10 semicolon=0 match=none
            6c consumed=1 overconsumed=11 semicolon=0 match=none
            64 consumed=1 overconsumed=12 semicolon=0 match=none
            65 consumed=1 overconsumed=13 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=2242/824
            """)
        check_matcher("amp;", """
            61 consumed=1 overconsumed=1 semicolon=0 match=none
            6d consumed=1 overconsumed=2 semicolon=0 match=none
            70 consumed=1 overconsumed=0 semicolon=0 match=26/0
            3b consumed=1 overconsumed=0 semicolon=1 match=26/0
            """)
        check_matcher("AMP", """
            41 consumed=1 overconsumed=1 semicolon=0 match=none
            4d consumed=1 overconsumed=2 semicolon=0 match=none
            50 consumed=1 overconsumed=0 semicolon=0 match=26/0
            """)
        check_matcher("ampx;", """
            61 consumed=1 overconsumed=1 semicolon=0 match=none
            6d consumed=1 overconsumed=2 semicolon=0 match=none
            70 consumed=1 overconsumed=0 semicolon=0 match=26/0
            78 consumed=0 overconsumed=0 semicolon=0 match=26/0
            3b consumed=1 overconsumed=0 semicolon=1 match=26/0
            """)
        check_matcher("zwnj;", """
            7a consumed=1 overconsumed=1 semicolon=0 match=none
            77 consumed=1 overconsumed=2 semicolon=0 match=none
            6e consumed=1 overconsumed=3 semicolon=0 match=none
            6a consumed=1 overconsumed=4 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=200c/0
            """)
        check_matcher("Zscr;", """
            5a consumed=1 overconsumed=1 semicolon=0 match=none
            73 consumed=1 overconsumed=2 semicolon=0 match=none
            63 consumed=1 overconsumed=3 semicolon=0 match=none
            72 consumed=1 overconsumed=4 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=1d4b5/0
            """)
        check_matcher("a", """
            61 consumed=1 overconsumed=1 semicolon=0 match=none
            """)
        check_matcher("b", """
            62 consumed=1 overconsumed=1 semicolon=0 match=none
            """)
        check_matcher("Z;", """
            5a consumed=1 overconsumed=1 semicolon=0 match=none
            3b consumed=0 overconsumed=1 semicolon=0 match=none
            """)
        check_matcher("nGt;", """
            6e consumed=1 overconsumed=1 semicolon=0 match=none
            47 consumed=1 overconsumed=2 semicolon=0 match=none
            74 consumed=1 overconsumed=3 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=226b/8402
            """)
        check_matcher("acE;", """
            61 consumed=1 overconsumed=1 semicolon=0 match=none
            63 consumed=1 overconsumed=2 semicolon=0 match=none
            45 consumed=1 overconsumed=3 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=223e/819
            """)
        check_matcher("ThickSpace;", """
            54 consumed=1 overconsumed=1 semicolon=0 match=none
            68 consumed=1 overconsumed=2 semicolon=0 match=none
            69 consumed=1 overconsumed=3 semicolon=0 match=none
            63 consumed=1 overconsumed=4 semicolon=0 match=none
            6b consumed=1 overconsumed=5 semicolon=0 match=none
            53 consumed=1 overconsumed=6 semicolon=0 match=none
            70 consumed=1 overconsumed=7 semicolon=0 match=none
            61 consumed=1 overconsumed=8 semicolon=0 match=none
            63 consumed=1 overconsumed=9 semicolon=0 match=none
            65 consumed=1 overconsumed=10 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=205f/8202
            """)
        check_matcher("fjlig;", """
            66 consumed=1 overconsumed=1 semicolon=0 match=none
            6a consumed=1 overconsumed=2 semicolon=0 match=none
            6c consumed=1 overconsumed=3 semicolon=0 match=none
            69 consumed=1 overconsumed=4 semicolon=0 match=none
            67 consumed=1 overconsumed=5 semicolon=0 match=none
            3b consumed=1 overconsumed=0 semicolon=1 match=66/106
            """)