Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.1

0.6.1 / tests / web_test / driver_report.lucb

#==============================================================================================
#
#   driver_report - expected failures, the results directory and the summary
#
#   DESCRIPTION:
#       The expected failures (DESIGN.md §6.2) are tests/expected_failures/{layout,ref,crash,screenshot}.txt:
#       one line per test that is known to fail, `PATH REASON` with an optional `# NOTE`, PATH
#       relative to the test root as the runner prints it. The reasons:
#
#         unported   the test reaches a function that is still a stub (`unported: NAME`; the
#                    note names the first one it reached)
#         noscript   the test has <noscript>, whose parsing needs scripting (phase 3)
#         script     the test needs script; resource: it needs a resource (phase 2 and 3)
#         font       the test depends on fonts the test fonts do not reproduce
#         libm       last-bit differences of the platform's libm
#         raster     a Screenshot test whose picture is right but whose pixels are not Skia's
#                    (the anti-aliasing of glyphs and edges, gradient dithering, blur), by
#                    more than its fuzzy configuration allows (DESIGN.md §6.4, §7.3)
#         bug        anything else (a difference, a crash, a timeout): to be investigated
#
#       A run fails on an unexpected failure (a failing test that is not listed) and on an
#       unexpected pass (a listed test that passes), so the lists only shrink. --update-failures
#       rewrites the lists from the run: every failing test is listed, with the reason it had
#       when that reason was given by hand, otherwise `unported` or `noscript` or `bug`, and a
#       note of how it failed (for a Layout test, also of how far it got without rendering); the
#       entries of tests the run did not run are kept.
#
#       The results directory (build/web_test_results by default) holds, for every test that did
#       not pass, PATH.log (how it ended and the worker's output), for a Layout test
#       PATH.actual.txt (what the worker dumped, also partially) and for a failed Ref or
#       Screenshot comparison PATH.actual.png, PATH.expected.png and PATH.diff.png (written by the
#       worker), plus results.tsv (one line per test, a failed comparison with its statistics)
#       and summary.txt (what the run printed).
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: Expected failures =======================================================================

## One line of an expected-failures list.
struct ExpectedFailure:
    var path: str
    var reason: str
    var note: str

## The expected-failures list of `mode` in `directory` (empty when there is none).
func load_expected_failures(directory: str, mode: TestMode) -> ak.Vector[ExpectedFailure]:
    let text = read_text_file(cat([directory, "/", test_mode_name(mode), ".txt"])) else return ak.Vector[ExpectedFailure]()
    return parse_expected_failures(text)

## The entries of the text of an expected-failures list.
func parse_expected_failures(text: str) -> ak.Vector[ExpectedFailure]:
    var entries: ak.Vector[ExpectedFailure]
    var lines = strings.SplitIterator.over(text, "\n") catch failure: trap("out of memory")
    while let raw_line = lines.next():
        var line = strings.trim(raw_line)
        var note = ""
        if let parts = strings.split_once(line, "#"):
            line = strings.trim(parts.0)
            note = strings.trim(parts.1)
        if line.length == 0:
            continue
        var path = line
        var reason = "bug"
        if let fields = strings.split_once(line, " "):
            path = strings.trim(fields.0)
            reason = strings.trim(fields.1)
        ak.vector_append_t(&entries, ExpectedFailure(path = path, reason = reason, note = note))
    return entries

## The entry of `path` in `entries`, if it has one.
func find_expected_failure(entries: const ak.Vector[ExpectedFailure]*, path: str) -> const ExpectedFailure*?:
    for entry in &ak.vector_span_const(entries):
        if entry.path == path:
            return entry
    return none

## Whether a result counts as passing.
func is_pass(result: const TestResult*) -> bool:
    return result.outcome == Outcome.pass

## Whether a result counts as failing (skipped tests are neither).
func is_failure(result: const TestResult*) -> bool:
    return result.outcome != Outcome.pass and result.outcome != Outcome.skipped

## Whether `text` contains `needle`, ignoring ASCII case.
func contains_ignoring_case(text: str, needle: str) -> bool:
    let bytes = text.bytes
    let pattern = needle.bytes
    if pattern.length > bytes.length:
        return false
    for start in 0..<bytes.length - pattern.length + 1:
        var matches = true
        for offset in 0..<pattern.length:
            var byte = bytes[start + offset]
            if byte >= 'A' and byte <= 'Z':
                byte += 32
            if byte != pattern[offset]:
                matches = false
                break
        if matches:
            return true
    return false

## The `unported…` part of a trap message, if it is one.
func unported_name(message: str) -> str?:
    if let at = strings.find(message, "unported"):
        return (str)message.bytes[at..]
    return none

## The reason and note --update-failures gives a failing test.
func classify_failure(root: str, test_case: const Test*, result: const TestResult*) -> (str, str):
    let note = cat([describe_failure(result), without_rendering_note(result)])
    if result.outcome == Outcome.trapped:
        if let name = unported_name(result.trap_message):
            return ("unported", cat([name, without_rendering_note(result)]))
    if let input = read_text_file(cat([root, "/", test_case.relative_path])):
        if contains_ignoring_case(input, "<noscript"):
            return ("noscript", note)
    return ("bug", note)

## What the second run without rendering gave, for a note (empty without one).
func without_rendering_note(result: const TestResult*) -> str:
    if not result.retried_without_rendering:
        return ""
    if result.dumps_match_without_rendering:
        return "; without rendering the dumps match"
    if result.trap_message_without_rendering.length > 0:
        let name = unported_name(result.trap_message_without_rendering) else result.trap_message_without_rendering
        return cat(["; without rendering: ", name])
    if result.layout_tree_matches_without_rendering:
        return "; without rendering the layout tree matches"
    return "; without rendering the dumps differ"

## How a test failed, in a few words.
func describe_failure(result: const TestResult*) -> str:
    return match result.outcome:
        .trapped => cat(["trap: ", result.trap_message])
        .timeout => "timeout"
        .crash => cat(["crash: ", result.detail])
        .fail => result.detail
        .pass, .skipped => ""

## Whether a reason is one --update-failures gives (and so recomputes).
func is_automatic_reason(reason: str) -> bool:
    return reason == "unported" or reason == "noscript" or reason == "bug"

## Rewrites the expected-failures list of `mode` from the results (see the header).
func update_expected_failures(options: const Options*, mode: TestMode, tests: const Test[], results: const TestResult[],
                              previous: const ak.Vector[ExpectedFailure]*):
    var entries: ak.Vector[ExpectedFailure]
    # Entries of tests this run did not run stay.
    for entry in ak.vector_span_const(previous):
        var ran = false
        for index in 0..<tests.length:
            if tests[index].mode == mode and tests[index].relative_path == entry.path and results[index].outcome != Outcome.skipped:
                ran = true
                break
        if not ran:
            ak.vector_append_t(&entries, entry)
    for index in 0..<tests.length:
        let test_case = &tests[index]
        let result = &results[index]
        if test_case.mode != mode or not is_failure(result):
            continue
        var (reason, note) = classify_failure(options.root, test_case, result)
        if let old = find_expected_failure(previous, test_case.relative_path):
            if not is_automatic_reason(old.reason):
                reason = old.reason
        ak.vector_append_t(&entries, ExpectedFailure(path = test_case.relative_path, reason = reason, note = note))

    # In path order.
    let span = ak.vector_span(&entries)
    for i in 1..<span.length:
        var j = i
        while j > 0 and span[j].path < span[j - 1].path:
            let saved = span[j]
            span[j] = span[j - 1]
            span[j - 1] = saved
            j -= 1

    var text: collections.Buffer
    text.put_text(cat(["# Tests of web_test's ", test_mode_name(mode), " suite that are known to fail: PATH REASON # NOTE.\n"])) catch failure: trap("out of memory")
    text.put_text("# Reasons and how the list is kept: tests/web_test/driver_report.lucb (written by web_test --update-failures).\n") catch failure: trap("out of memory")
    for entry in span:
        text.put_text(cat([entry.path, " ", entry.reason])) catch failure: trap("out of memory")
        if entry.note.length > 0:
            text.put_text(cat([" # ", one_line(entry.note)])) catch failure: trap("out of memory")
        text.put_text("\n") catch failure: trap("out of memory")
    make_directories(options.expected_failures)
    files.write(c_path(cat([options.expected_failures, "/", test_mode_name(mode), ".txt"])), text.view()) catch failure: trap("web_test: cannot write the expected failures")

## `text` up to its first newline.
func one_line(text: str) -> str:
    if let at = strings.find(text, "\n"):
        return (str)text.bytes[..<at]
    return text

# mark: The results directory ===================================================================

## Creates `path` and its missing parents.
func make_directories(path: str):
    var index: usize = 1
    while index <= path.length:
        if index == path.length or path.bytes[index] == '/':
            let prefix = (str)path.bytes[..<index]
            files.ensure_directory(prefix) catch failure: trap(f"web_test: cannot create {prefix}")
        index += 1

## The directory part of `path`.
func directory_of(path: str) -> str:
    if let slash = strings.last(path, '/'):
        return (str)path.bytes[..<slash]
    return "."

## Writes `text` to `path`, creating its directory.
func write_result_file(path: str, text: str):
    make_directories(directory_of(path))
    files.write(c_path(path), text.bytes) catch failure: trap(f"web_test: cannot write {path}")

## Writes what a test that did not pass left (see the header).
func write_failure_files(options: const Options*, test_case: const Test*, result: const TestResult*):
    let base = cat([options.results, "/", test_case.relative_path])
    write_result_file(cat([base, ".log"]), cat(["outcome: ", outcome_name(result.outcome), "\nstage: ", result.stage, "\ntrap: ", result.trap_message,
                                                "\ndetail: ", result.detail, "\nstatistics: ", result.statistics, "\n\n", result.log]))
    if test_case.mode == TestMode.layout and result.output.length > 0:
        write_result_file(cat([base, ".actual.txt"]), result.output)

# mark: The summary =============================================================================

## A message and how many tests stopped at it.
struct Count:
    var key: str
    var count: usize

## Counts `key` in `counts`.
func count_key(counts: ak.Vector[Count]*, key: str):
    for entry in &ak.vector_span(counts):
        if entry.key == key:
            entry.count += 1
            return
    ak.vector_append_t(counts, Count(key = key, count = 1))

## `counts`, most frequent first (ties in first-seen order).
func sorted_counts(counts: ak.Vector[Count]*) -> Count[]:
    let span = ak.vector_span(counts)
    for i in 1..<span.length:
        var j = i
        while j > 0 and span[j].count > span[j - 1].count:
            let saved = span[j]
            span[j] = span[j - 1]
            span[j - 1] = saved
            j -= 1
    return span

## The summary of the run (printed and written to summary.txt), and whether it found unexpected
## failures or passes.
func summarize(options: const Options*, tests: const Test[], results: const TestResult[], expected: const ak.Vector[ExpectedFailure][]) -> (str, bool):
    var out: collections.Buffer
    var unexpected_failures: collections.Buffer
    var unexpected_passes: collections.Buffer
    var unexpected_failure_count: usize = 0
    var unexpected_pass_count: usize = 0
    var traps: ak.Vector[Count]
    var stages: ak.Vector[Count]

    for mode in test_modes:
        if not options.suites[(usize)test_mode_index(mode)]:
            continue
        var counts: usize[6] = [0, 0, 0, 0, 0, 0]
        var total: usize = 0
        var http: usize = 0
        var layout_trees: usize = 0
        var retried: usize = 0
        var retried_dumps: usize = 0
        var retried_trees: usize = 0
        var retried_traps: ak.Vector[Count]
        for index in 0..<tests.length:
            let test_case = &tests[index]
            if test_case.mode != mode:
                continue
            let result = &results[index]
            total += 1
            counts[(usize)outcome_index(result.outcome)] += 1
            if result.skip == SkipReason.needs_http_server:
                http += 1
            if result.layout_tree_matches:
                layout_trees += 1
            if result.retried_without_rendering:
                retried += 1
                if result.dumps_match_without_rendering: retried_dumps += 1
                if result.layout_tree_matches_without_rendering: retried_trees += 1
                if result.trap_message_without_rendering.length > 0:
                    count_key(&retried_traps, unported_name(result.trap_message_without_rendering) else result.trap_message_without_rendering)
            if result.outcome == Outcome.trapped:
                count_key(&traps, unported_name(result.trap_message) else result.trap_message)
            if is_failure(result):
                count_key(&stages, cat([test_mode_name(mode), ": ", result.stage]))
            let listed = find_expected_failure(&expected[(usize)test_mode_index(mode)], test_case.relative_path) != none
            if is_failure(result) and not listed:
                unexpected_failure_count += 1
                unexpected_failures.put_text(cat(["  ", test_case.relative_path, "  ", one_line(describe_failure(result)), "\n"])) catch failure: trap("out of memory")
            if is_pass(result) and listed:
                unexpected_pass_count += 1
                unexpected_passes.put_text(cat(["  ", test_case.relative_path, "\n"])) catch failure: trap("out of memory")
        if total == 0:
            continue
        out.put_text(cat([test_mode_name(mode), ": ", format_count((u64)total), " tests: ", format_count((u64)counts[0]), " pass, ",
                          format_count((u64)counts[1]), " fail, ", format_count((u64)counts[2]), " trap, ", format_count((u64)counts[3]), " crash, ",
                          format_count((u64)counts[4]), " timeout, ", format_count((u64)counts[5]), " skipped (", format_count((u64)http),
                          " need an HTTP server)\n"])) catch failure: trap("out of memory")
        if mode == TestMode.layout:
            out.put_text(cat(["  layout tree matches: ", format_count((u64)layout_trees), "\n"])) catch failure: trap("out of memory")
            if retried > 0:
                out.put_text(cat(["  without rendering (the ", format_count((u64)retried), " tests that trapped or crashed, again with --no-rendering): ",
                                  format_count((u64)retried_dumps), " match all three dumps, ", format_count((u64)retried_trees),
                                  " match the layout tree\n"])) catch failure: trap("out of memory")
                let sorted = sorted_counts(&retried_traps)
                for index in 0..<sorted.length:
                    if index == 10:
                        break
                    out.put_text(cat(["  ", pad_count(sorted[index].count), "  ", sorted[index].key, " (without rendering)\n"])) catch failure: trap("out of memory")

    if not ak.vector_is_empty(&stages):
        out.put_text("\nWhere the failing tests stopped (suite: last stage reached):\n") catch failure: trap("out of memory")
        for entry in sorted_counts(&stages):
            out.put_text(cat([pad_count(entry.count), "  ", entry.key, "\n"])) catch failure: trap("out of memory")
    if not ak.vector_is_empty(&traps):
        out.put_text("\nTop traps:\n") catch failure: trap("out of memory")
        let sorted = sorted_counts(&traps)
        for index in 0..<sorted.length:
            if index == 40:
                out.put_text(cat(["  ... and ", format_count((u64)(sorted.length - 40)), " more\n"])) catch failure: trap("out of memory")
                break
            out.put_text(cat([pad_count(sorted[index].count), "  ", sorted[index].key, "\n"])) catch failure: trap("out of memory")
    if unexpected_failure_count > 0:
        out.put_text(cat(["\nUnexpected failures (", format_count((u64)unexpected_failure_count), "):\n", unexpected_failures.text()])) catch failure: trap("out of memory")
    if unexpected_pass_count > 0:
        out.put_text(cat(["\nUnexpected passes (", format_count((u64)unexpected_pass_count), "), remove them from tests/expected_failures:\n",
                          unexpected_passes.text()])) catch failure: trap("out of memory")
    let ok = unexpected_failure_count == 0 and unexpected_pass_count == 0
    out.put_text("\nweb_test: OK (no unexpected failures or passes)\n" if ok else "\nweb_test: FAILED\n") catch failure: trap("out of memory")
    let summary = out.text()
    return (summary, ok)

## The index of an outcome in the summary's counts.
func outcome_index(outcome: Outcome) -> u8:
    return match outcome:
        .pass => 0
        .fail => 1
        .trapped => 2
        .crash => 3
        .timeout => 4
        .skipped => 5

## A count right-aligned in six columns.
func pad_count(count: usize) -> str:
    let digits = format_count((u64)count)
    var spaces = ""
    if digits.length < 6:
        spaces = (str)"      ".bytes[..<6 - digits.length]
    return cat([spaces, digits])

# mark: The driver ==============================================================================

## web_test's driver mode (see module.lucb): answers 0 when the run had no unexpected failure or
## pass (or rewrote the lists), 1 otherwise.
func driver_main(options: const Options*) -> i32:
    let collected = collect_selected_tests(options)
    let tests = ak.vector_span_const(&collected)
    if tests.length == 0:
        io.stderr().write("web_test: No tests found matching filter\n") catch failure: return 1
        return 1
    let config = load_test_config(options.root)
    var skips: ak.Vector[SkipReason]
    for test_case in &tests:
        ak.vector_append_t(&skips, skip_reason(&config, test_case))

    if is_directory(options.results):
        files.remove_tree(c_path(options.results)) catch failure: trap(f"web_test: cannot clear {options.results}")
    let work = cat([options.results, "/.work"])
    make_directories(work)

    print(f"web_test: running {tests.length} tests, {options.jobs} at a time")
    let started = time.now()
    let results_vector = run_tests(options, tests, ak.vector_span_const(&skips), work)
    let results = ak.vector_span_const(&results_vector)
    let seconds = time.since(started) // 1000000000

    var expected: ak.Vector[ExpectedFailure][4]
    for mode in test_modes:
        expected[(usize)test_mode_index(mode)] = load_expected_failures(options.expected_failures, mode)

    var table: collections.Buffer
    for index in 0..<tests.length:
        let test_case = &tests[index]
        let result = &results[index]
        var detail = one_line(describe_failure(result))
        if result.statistics.length > 0:
            detail = cat([detail, ": ", one_line(result.statistics)])
        table.put_text(cat([test_case.relative_path, "\t", test_mode_name(test_case.mode), "\t", outcome_name(result.outcome), "\t", result.stage, "\t",
                            detail, "\n"])) catch failure: trap("out of memory")
        if is_failure(result):
            write_failure_files(options, test_case, result)
    write_result_file(cat([options.results, "/results.tsv"]), table.text())
    files.remove_tree(c_path(work)) catch failure: trap(f"web_test: cannot remove {work}")

    let (summary, ok) = summarize(options, tests, results, expected)
    let timed = cat([summary, "(", format_count(seconds), " s; results in ", options.results, ")\n"])
    io.stdout().write(timed.bytes) catch failure: return 1
    write_result_file(cat([options.results, "/summary.txt"]), timed)

    if options.update_failures:
        for mode in test_modes:
            if options.suites[(usize)test_mode_index(mode)]:
                update_expected_failures(options, mode, tests, results, &expected[(usize)test_mode_index(mode)])
        print(f"web_test: rewrote the expected failures in {options.expected_failures}")
        return 0
    return 0 if ok else 1