dymokomi/luce-browser-engine
Source of release 0.6.0
0.6.0 / tests / web_test / driver_report.lucb
#==============================================================================================
#
# driver_report - expected failures, the results directory and the summary
#
# DESCRIPTION:
# The expected failures (DESIGN.md §6.2) are tests/expected_failures/{layout,ref,crash,screenshot}.txt:
# one line per test that is known to fail, `PATH REASON` with an optional `# NOTE`, PATH
# relative to the test root as the runner prints it. The reasons:
#
# unported the test reaches a function that is still a stub (`unported: NAME`; the
# note names the first one it reached)
# noscript the test has <noscript>, whose parsing needs scripting (phase 3)
# script the test needs script; resource: it needs a resource (phase 2 and 3)
# font the test depends on fonts the test fonts do not reproduce
# libm last-bit differences of the platform's libm
# raster a Screenshot test whose picture is right but whose pixels are not Skia's
# (the anti-aliasing of glyphs and edges, gradient dithering, blur), by
# more than its fuzzy configuration allows (DESIGN.md §6.4, §7.3)
# bug anything else (a difference, a crash, a timeout): to be investigated
#
# A run fails on an unexpected failure (a failing test that is not listed) and on an
# unexpected pass (a listed test that passes), so the lists only shrink. --update-failures
# rewrites the lists from the run: every failing test is listed, with the reason it had
# when that reason was given by hand, otherwise `unported` or `noscript` or `bug`, and a
# note of how it failed (for a Layout test, also of how far it got without rendering); the
# entries of tests the run did not run are kept.
#
# The results directory (build/web_test_results by default) holds, for every test that did
# not pass, PATH.log (how it ended and the worker's output), for a Layout test
# PATH.actual.txt (what the worker dumped, also partially) and for a failed Ref or
# Screenshot comparison PATH.actual.png, PATH.expected.png and PATH.diff.png (written by the
# worker), plus results.tsv (one line per test, a failed comparison with its statistics)
# and summary.txt (what the run printed).
#
# Copyright (c) 2026 the luce-browser authors.
# SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================
# mark: Expected failures =======================================================================
## One line of an expected-failures list.
struct ExpectedFailure:
var path: str
var reason: str
var note: str
## The expected-failures list of `mode` in `directory` (empty when there is none).
func load_expected_failures(directory: str, mode: TestMode) -> ak.Vector[ExpectedFailure]:
let text = read_text_file(cat([directory, "/", test_mode_name(mode), ".txt"])) else return ak.Vector[ExpectedFailure]()
return parse_expected_failures(text)
## The entries of the text of an expected-failures list.
func parse_expected_failures(text: str) -> ak.Vector[ExpectedFailure]:
var entries: ak.Vector[ExpectedFailure]
var lines = strings.SplitIterator.over(text, "\n") catch failure: trap("out of memory")
while let raw_line = lines.next():
var line = strings.trim(raw_line)
var note = ""
if let parts = strings.split_once(line, "#"):
line = strings.trim(parts.0)
note = strings.trim(parts.1)
if line.length == 0:
continue
var path = line
var reason = "bug"
if let fields = strings.split_once(line, " "):
path = strings.trim(fields.0)
reason = strings.trim(fields.1)
ak.vector_append_t(&entries, ExpectedFailure(path = path, reason = reason, note = note))
return entries
## The entry of `path` in `entries`, if it has one.
func find_expected_failure(entries: const ak.Vector[ExpectedFailure]*, path: str) -> const ExpectedFailure*?:
for entry in &ak.vector_span_const(entries):
if entry.path == path:
return entry
return none
## Whether a result counts as passing.
func is_pass(result: const TestResult*) -> bool:
return result.outcome == Outcome.pass
## Whether a result counts as failing (skipped tests are neither).
func is_failure(result: const TestResult*) -> bool:
return result.outcome != Outcome.pass and result.outcome != Outcome.skipped
## Whether `text` contains `needle`, ignoring ASCII case.
func contains_ignoring_case(text: str, needle: str) -> bool:
let bytes = text.bytes
let pattern = needle.bytes
if pattern.length > bytes.length:
return false
for start in 0..<bytes.length - pattern.length + 1:
var matches = true
for offset in 0..<pattern.length:
var byte = bytes[start + offset]
if byte >= 'A' and byte <= 'Z':
byte += 32
if byte != pattern[offset]:
matches = false
break
if matches:
return true
return false
## The `unported…` part of a trap message, if it is one.
func unported_name(message: str) -> str?:
if let at = strings.find(message, "unported"):
return (str)message.bytes[at..]
return none
## The reason and note --update-failures gives a failing test.
func classify_failure(root: str, test_case: const Test*, result: const TestResult*) -> (str, str):
let note = cat([describe_failure(result), without_rendering_note(result)])
if result.outcome == Outcome.trapped:
if let name = unported_name(result.trap_message):
return ("unported", cat([name, without_rendering_note(result)]))
if let input = read_text_file(cat([root, "/", test_case.relative_path])):
if contains_ignoring_case(input, "<noscript"):
return ("noscript", note)
return ("bug", note)
## What the second run without rendering gave, for a note (empty without one).
func without_rendering_note(result: const TestResult*) -> str:
if not result.retried_without_rendering:
return ""
if result.dumps_match_without_rendering:
return "; without rendering the dumps match"
if result.trap_message_without_rendering.length > 0:
let name = unported_name(result.trap_message_without_rendering) else result.trap_message_without_rendering
return cat(["; without rendering: ", name])
if result.layout_tree_matches_without_rendering:
return "; without rendering the layout tree matches"
return "; without rendering the dumps differ"
## How a test failed, in a few words.
func describe_failure(result: const TestResult*) -> str:
return match result.outcome:
.trapped => cat(["trap: ", result.trap_message])
.timeout => "timeout"
.crash => cat(["crash: ", result.detail])
.fail => result.detail
.pass, .skipped => ""
## Whether a reason is one --update-failures gives (and so recomputes).
func is_automatic_reason(reason: str) -> bool:
return reason == "unported" or reason == "noscript" or reason == "bug"
## Rewrites the expected-failures list of `mode` from the results (see the header).
func update_expected_failures(options: const Options*, mode: TestMode, tests: const Test[], results: const TestResult[],
previous: const ak.Vector[ExpectedFailure]*):
var entries: ak.Vector[ExpectedFailure]
# Entries of tests this run did not run stay.
for entry in ak.vector_span_const(previous):
var ran = false
for index in 0..<tests.length:
if tests[index].mode == mode and tests[index].relative_path == entry.path and results[index].outcome != Outcome.skipped:
ran = true
break
if not ran:
ak.vector_append_t(&entries, entry)
for index in 0..<tests.length:
let test_case = &tests[index]
let result = &results[index]
if test_case.mode != mode or not is_failure(result):
continue
var (reason, note) = classify_failure(options.root, test_case, result)
if let old = find_expected_failure(previous, test_case.relative_path):
if not is_automatic_reason(old.reason):
reason = old.reason
ak.vector_append_t(&entries, ExpectedFailure(path = test_case.relative_path, reason = reason, note = note))
# In path order.
let span = ak.vector_span(&entries)
for i in 1..<span.length:
var j = i
while j > 0 and span[j].path < span[j - 1].path:
let saved = span[j]
span[j] = span[j - 1]
span[j - 1] = saved
j -= 1
var text: collections.Buffer
text.put_text(cat(["# Tests of web_test's ", test_mode_name(mode), " suite that are known to fail: PATH REASON # NOTE.\n"])) catch failure: trap("out of memory")
text.put_text("# Reasons and how the list is kept: tests/web_test/driver_report.lucb (written by web_test --update-failures).\n") catch failure: trap("out of memory")
for entry in span:
text.put_text(cat([entry.path, " ", entry.reason])) catch failure: trap("out of memory")
if entry.note.length > 0:
text.put_text(cat([" # ", one_line(entry.note)])) catch failure: trap("out of memory")
text.put_text("\n") catch failure: trap("out of memory")
make_directories(options.expected_failures)
files.write(c_path(cat([options.expected_failures, "/", test_mode_name(mode), ".txt"])), text.view()) catch failure: trap("web_test: cannot write the expected failures")
## `text` up to its first newline.
func one_line(text: str) -> str:
if let at = strings.find(text, "\n"):
return (str)text.bytes[..<at]
return text
# mark: The results directory ===================================================================
## Creates `path` and its missing parents.
func make_directories(path: str):
var index: usize = 1
while index <= path.length:
if index == path.length or path.bytes[index] == '/':
let prefix = (str)path.bytes[..<index]
files.ensure_directory(prefix) catch failure: trap(f"web_test: cannot create {prefix}")
index += 1
## The directory part of `path`.
func directory_of(path: str) -> str:
if let slash = strings.last(path, '/'):
return (str)path.bytes[..<slash]
return "."
## Writes `text` to `path`, creating its directory.
func write_result_file(path: str, text: str):
make_directories(directory_of(path))
files.write(c_path(path), text.bytes) catch failure: trap(f"web_test: cannot write {path}")
## Writes what a test that did not pass left (see the header).
func write_failure_files(options: const Options*, test_case: const Test*, result: const TestResult*):
let base = cat([options.results, "/", test_case.relative_path])
write_result_file(cat([base, ".log"]), cat(["outcome: ", outcome_name(result.outcome), "\nstage: ", result.stage, "\ntrap: ", result.trap_message,
"\ndetail: ", result.detail, "\nstatistics: ", result.statistics, "\n\n", result.log]))
if test_case.mode == TestMode.layout and result.output.length > 0:
write_result_file(cat([base, ".actual.txt"]), result.output)
# mark: The summary =============================================================================
## A message and how many tests stopped at it.
struct Count:
var key: str
var count: usize
## Counts `key` in `counts`.
func count_key(counts: ak.Vector[Count]*, key: str):
for entry in &ak.vector_span(counts):
if entry.key == key:
entry.count += 1
return
ak.vector_append_t(counts, Count(key = key, count = 1))
## `counts`, most frequent first (ties in first-seen order).
func sorted_counts(counts: ak.Vector[Count]*) -> Count[]:
let span = ak.vector_span(counts)
for i in 1..<span.length:
var j = i
while j > 0 and span[j].count > span[j - 1].count:
let saved = span[j]
span[j] = span[j - 1]
span[j - 1] = saved
j -= 1
return span
## The summary of the run (printed and written to summary.txt), and whether it found unexpected
## failures or passes.
func summarize(options: const Options*, tests: const Test[], results: const TestResult[], expected: const ak.Vector[ExpectedFailure][]) -> (str, bool):
var out: collections.Buffer
var unexpected_failures: collections.Buffer
var unexpected_passes: collections.Buffer
var unexpected_failure_count: usize = 0
var unexpected_pass_count: usize = 0
var traps: ak.Vector[Count]
var stages: ak.Vector[Count]
for mode in test_modes:
if not options.suites[(usize)test_mode_index(mode)]:
continue
var counts: usize[6] = [0, 0, 0, 0, 0, 0]
var total: usize = 0
var http: usize = 0
var layout_trees: usize = 0
var retried: usize = 0
var retried_dumps: usize = 0
var retried_trees: usize = 0
var retried_traps: ak.Vector[Count]
for index in 0..<tests.length:
let test_case = &tests[index]
if test_case.mode != mode:
continue
let result = &results[index]
total += 1
counts[(usize)outcome_index(result.outcome)] += 1
if result.skip == SkipReason.needs_http_server:
http += 1
if result.layout_tree_matches:
layout_trees += 1
if result.retried_without_rendering:
retried += 1
if result.dumps_match_without_rendering: retried_dumps += 1
if result.layout_tree_matches_without_rendering: retried_trees += 1
if result.trap_message_without_rendering.length > 0:
count_key(&retried_traps, unported_name(result.trap_message_without_rendering) else result.trap_message_without_rendering)
if result.outcome == Outcome.trapped:
count_key(&traps, unported_name(result.trap_message) else result.trap_message)
if is_failure(result):
count_key(&stages, cat([test_mode_name(mode), ": ", result.stage]))
let listed = find_expected_failure(&expected[(usize)test_mode_index(mode)], test_case.relative_path) != none
if is_failure(result) and not listed:
unexpected_failure_count += 1
unexpected_failures.put_text(cat([" ", test_case.relative_path, " ", one_line(describe_failure(result)), "\n"])) catch failure: trap("out of memory")
if is_pass(result) and listed:
unexpected_pass_count += 1
unexpected_passes.put_text(cat([" ", test_case.relative_path, "\n"])) catch failure: trap("out of memory")
if total == 0:
continue
out.put_text(cat([test_mode_name(mode), ": ", format_count((u64)total), " tests: ", format_count((u64)counts[0]), " pass, ",
format_count((u64)counts[1]), " fail, ", format_count((u64)counts[2]), " trap, ", format_count((u64)counts[3]), " crash, ",
format_count((u64)counts[4]), " timeout, ", format_count((u64)counts[5]), " skipped (", format_count((u64)http),
" need an HTTP server)\n"])) catch failure: trap("out of memory")
if mode == TestMode.layout:
out.put_text(cat([" layout tree matches: ", format_count((u64)layout_trees), "\n"])) catch failure: trap("out of memory")
if retried > 0:
out.put_text(cat([" without rendering (the ", format_count((u64)retried), " tests that trapped or crashed, again with --no-rendering): ",
format_count((u64)retried_dumps), " match all three dumps, ", format_count((u64)retried_trees),
" match the layout tree\n"])) catch failure: trap("out of memory")
let sorted = sorted_counts(&retried_traps)
for index in 0..<sorted.length:
if index == 10:
break
out.put_text(cat([" ", pad_count(sorted[index].count), " ", sorted[index].key, " (without rendering)\n"])) catch failure: trap("out of memory")
if not ak.vector_is_empty(&stages):
out.put_text("\nWhere the failing tests stopped (suite: last stage reached):\n") catch failure: trap("out of memory")
for entry in sorted_counts(&stages):
out.put_text(cat([pad_count(entry.count), " ", entry.key, "\n"])) catch failure: trap("out of memory")
if not ak.vector_is_empty(&traps):
out.put_text("\nTop traps:\n") catch failure: trap("out of memory")
let sorted = sorted_counts(&traps)
for index in 0..<sorted.length:
if index == 40:
out.put_text(cat([" ... and ", format_count((u64)(sorted.length - 40)), " more\n"])) catch failure: trap("out of memory")
break
out.put_text(cat([pad_count(sorted[index].count), " ", sorted[index].key, "\n"])) catch failure: trap("out of memory")
if unexpected_failure_count > 0:
out.put_text(cat(["\nUnexpected failures (", format_count((u64)unexpected_failure_count), "):\n", unexpected_failures.text()])) catch failure: trap("out of memory")
if unexpected_pass_count > 0:
out.put_text(cat(["\nUnexpected passes (", format_count((u64)unexpected_pass_count), "), remove them from tests/expected_failures:\n",
unexpected_passes.text()])) catch failure: trap("out of memory")
let ok = unexpected_failure_count == 0 and unexpected_pass_count == 0
out.put_text("\nweb_test: OK (no unexpected failures or passes)\n" if ok else "\nweb_test: FAILED\n") catch failure: trap("out of memory")
let summary = out.text()
return (summary, ok)
## The index of an outcome in the summary's counts.
func outcome_index(outcome: Outcome) -> u8:
return match outcome:
.pass => 0
.fail => 1
.trapped => 2
.crash => 3
.timeout => 4
.skipped => 5
## A count right-aligned in six columns.
func pad_count(count: usize) -> str:
let digits = format_count((u64)count)
var spaces = ""
if digits.length < 6:
spaces = (str)" ".bytes[..<6 - digits.length]
return cat([spaces, digits])
# mark: The driver ==============================================================================
## web_test's driver mode (see module.lucb): answers 0 when the run had no unexpected failure or
## pass (or rewrote the lists), 1 otherwise.
func driver_main(options: const Options*) -> i32:
let collected = collect_selected_tests(options)
let tests = ak.vector_span_const(&collected)
if tests.length == 0:
io.stderr().write("web_test: No tests found matching filter\n") catch failure: return 1
return 1
let config = load_test_config(options.root)
var skips: ak.Vector[SkipReason]
for test_case in &tests:
ak.vector_append_t(&skips, skip_reason(&config, test_case))
if is_directory(options.results):
files.remove_tree(c_path(options.results)) catch failure: trap(f"web_test: cannot clear {options.results}")
let work = cat([options.results, "/.work"])
make_directories(work)
print(f"web_test: running {tests.length} tests, {options.jobs} at a time")
let started = time.now()
let results_vector = run_tests(options, tests, ak.vector_span_const(&skips), work)
let results = ak.vector_span_const(&results_vector)
let seconds = time.since(started) // 1000000000
var expected: ak.Vector[ExpectedFailure][4]
for mode in test_modes:
expected[(usize)test_mode_index(mode)] = load_expected_failures(options.expected_failures, mode)
var table: collections.Buffer
for index in 0..<tests.length:
let test_case = &tests[index]
let result = &results[index]
var detail = one_line(describe_failure(result))
if result.statistics.length > 0:
detail = cat([detail, ": ", one_line(result.statistics)])
table.put_text(cat([test_case.relative_path, "\t", test_mode_name(test_case.mode), "\t", outcome_name(result.outcome), "\t", result.stage, "\t",
detail, "\n"])) catch failure: trap("out of memory")
if is_failure(result):
write_failure_files(options, test_case, result)
write_result_file(cat([options.results, "/results.tsv"]), table.text())
files.remove_tree(c_path(work)) catch failure: trap(f"web_test: cannot remove {work}")
let (summary, ok) = summarize(options, tests, results, expected)
let timed = cat([summary, "(", format_count(seconds), " s; results in ", options.results, ")\n"])
io.stdout().write(timed.bytes) catch failure: return 1
write_result_file(cat([options.results, "/summary.txt"]), timed)
if options.update_failures:
for mode in test_modes:
if options.suites[(usize)test_mode_index(mode)]:
update_expected_failures(options, mode, tests, results, &expected[(usize)test_mode_index(mode)])
print(f"web_test: rewrote the expected failures in {options.expected_failures}")
return 0
return 0 if ok else 1