Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-css

Source of release 0.5.1

0.5.1 / src / css_syntax / tests_css_tokenizer.lucb

#==============================================================================================
#
#   tests_css_tokenizer - the CSS tokenizer corpus (Tests/LibWeb/CSSTokenizer)
#
#   DESCRIPTION:
#       The Luce driver of Ladybird's tokenizer corpus (tests/css_tokenizer, copied from the
#       donor) and of this port's extra corpus (tests/css_tokenizer_extra: edge cases whose
#       expectations were written by the donor's css-tokenizer program locally), done the way test-css-tokenizer.py
#       runs the css-tokenizer program (Tests/LibWeb/css-tokenizer.cpp): tokenize each
#       <corpus>/input/<name>.css in the encoding its <name>.css.encoding names
#       (utf-8 without one), print every token's to_debug_string() on its own line, and compare
#       with expected/<name>.txt, both stripped of surrounding whitespace. Paths are relative to
#       the package root, where test.sh runs.
#
#==============================================================================================

# mark: Reading the corpus ======================================================================

extern func fopen(path: const u8*, mode: const u8*) -> void*?
extern func fread(buffer: void*, size: usize, count: usize, stream: void*) -> usize
extern func fclose(stream: void*) -> i32

## The bytes of the file at `path` (a NUL-terminated path), or none when it cannot be opened.
func read_test_file(path: const u8*) -> u8[]?:
    let stream = fopen(path, "rb".bytes.data) else return none
    defer _ = fclose(stream)
    var data = new u8[1 << 12] else trap("out of memory")
    var used: usize = 0
    while true:
        if used == data.length:
            let grown = new u8[data.length * 2] else trap("out of memory")
            for index in 0..<used:
                grown[index] = data[index]
            data = grown
        let count = fread(&data[used], 1, data.length - used, stream)
        if count == 0:
            break
        used += count
    return data[..<used]

## `path` + NUL, for fopen.
func c_path(builder: ak.StringBuilder*, parts: const str[]) -> const u8*:
    for part in parts:
        ak.string_builder_append_string_view(builder, ak.sv(part))
    ak.string_builder_append_u8(builder, 0)
    return ak.string_builder_string_view(builder).m_characters else trap("empty path")

## `bytes` without the ASCII whitespace Python's str.strip() removes at both ends.
func strip_whitespace(bytes: const u8[]) -> const u8[]:
    var start: usize = 0
    var end = bytes.length
    while start < end and (bytes[start] == ' ' or bytes[start] == '\n' or bytes[start] == '\r' or bytes[start] == '\t' or bytes[start] == 0x0b or bytes[start] == 0x0c):
        start += 1
    while end > start and (bytes[end - 1] == ' ' or bytes[end - 1] == '\n' or bytes[end - 1] == '\r' or bytes[end - 1] == '\t' or bytes[end - 1] == 0x0b or bytes[end - 1] == 0x0c):
        end -= 1
    return bytes[start..<end]

# mark: The corpora =============================================================================

## Runs one corpus file; answers whether its tokens match the expected dump.
noinline func run_css_tokenizer_case(corpus: str, name: str) -> bool:
    var input_path = ak.string_builder_new()
    let input = read_test_file(c_path(&input_path, [corpus, "/input/", name, ".css"])) else trap("cannot read a corpus input")
    var encoding_path = ak.string_builder_new()
    var encoding = ak.sv("utf-8")
    if let encoding_file = read_test_file(c_path(&encoding_path, [corpus, "/input/", name, ".css.encoding"])):
        var view = ak.StringView()
        ak.string_view_construct_span_u8(&view, strip_whitespace(encoding_file))
        encoding = view
    var expected_path = ak.string_builder_new()
    let expected = read_test_file(c_path(&expected_path, [corpus, "/expected/", name, ".txt"])) else trap("cannot read a corpus expectation")

    var input_view = ak.StringView()
    ak.string_view_construct_span_u8(&input_view, input)
    let tokens = tokenizer_tokenize_string_view(input_view, encoding)
    var output = ak.string_builder_new()
    for token in &ak.vector_span_const(&tokens):
        let line = token_to_debug_string(token)
        string_builder_append_string(&output, &line)
        ak.string_builder_append_u8(&output, '\n')
    let actual_view = ak.string_builder_string_view(&output)
    let actual = strip_whitespace(u8[]((u8*)(actual_view.m_characters else trap("no output")), actual_view.m_length))
    let wanted = strip_whitespace(expected)
    if actual.length == wanted.length:
        var same = true
        for index in 0..<actual.length:
            if actual[index] != wanted[index]:
                same = false
                break
        if same:
            return true
    print(f"CSS tokens do not match for {corpus}/input/{name}.css!")
    print("--- expected")
    print((str)wanted)
    print("--- actual")
    print((str)actual)
    return false

## The corpus: every input file of Tests/LibWeb/CSSTokenizer/input.
let css_tokenizer_cases: str[15] = [
    "at-hash-function-delim",
    "basic-rules",
    "brackets-and-cdo-cdc",
    "comments-and-whitespace",
    "malformed-bad-string-recovery",
    "malformed-bad-url-recovery",
    "malformed-invalid-escape-extra-braces",
    "malformed-unterminated-comment",
    "malformed-unterminated-string-eof",
    "numeric-clamping",
    "numeric-tokens",
    "strings-and-escapes",
    "urls-and-bad-url",
    "utf-16le-crlf",
    "windows-1252-curly-quotes",
]

## This port's extra corpus: every input file of tests/css_tokenizer_extra/input.
let css_tokenizer_extra_cases: str[11] = [
    "columns-multibyte",
    "comments-edge",
    "empty",
    "filtered-code-points",
    "hashes-at-keywords",
    "idents-and-escapes",
    "numbers-edge",
    "shift-jis-basic",
    "strings-edge",
    "urls-edge",
    "utf-16be-basic",
]

noinline func test_css_tokenizer_corpus():
    var failed: usize = 0
    for name in css_tokenizer_cases:
        if not run_css_tokenizer_case("tests/css_tokenizer", name):
            failed += 1
    assert(failed == 0, "CSS tokenizer corpus failures")

noinline func test_css_tokenizer_extra_corpus():
    var failed: usize = 0
    for name in css_tokenizer_extra_cases:
        if not run_css_tokenizer_case("tests/css_tokenizer_extra", name):
            failed += 1
    assert(failed == 0, "CSS tokenizer extra corpus failures")

test "Tests/LibWeb/CSSTokenizer corpus":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_css_tokenizer_corpus()

test "tests/css_tokenizer_extra corpus (pinned to the donor's css-tokenizer)":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_css_tokenizer_extra_corpus()