Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.1

0.6.1 / tests / web_test / driver_run.lucb

#==============================================================================================
#
#   driver_run - running the tests: one worker process per test, in parallel, with a timeout
#
#   DESCRIPTION:
#       A luce-base trap cannot be caught and must not end the run (the luce-js test262 runner
#       splits for the same reason), so every test runs in a child process: this program with
#       `--one PATH --mode MODE --out FILE`, through luce-std's process.Command, which runs the
#       child on a worker thread, captures its standard output and error and can stop it. Up
#       to -j children run at once; one that is still running after --timeout seconds is
#       stopped (test-web's per-test timeout, 30 s by default).
#
#       A child's outcome: it exited 0 having written its result (then the result is judged:
#       a Layout dump against its expectation, "pass" or "fail" from a Ref, Crash or Screenshot
#       worker, which also writes the PNGs of a failed comparison into the results), it
#       trapped (its standard error holds "trap: FILE:LINE:COLUMN: MESSAGE"; an `unported:`
#       message names the next function the test needs), it died some other way (a signal, an
#       exit status), or it timed out. The last "web_test: stage NAME" line it wrote tells how
#       far it got. A trap's message is recorded with the working directory (the repository
#       root under luc test) left out of the paths it names, so the expected-failures notes do
#       not depend on the checkout.
#
#       A Layout test that trapped or crashed is run a second time with --no-rendering, which
#       dumps the page after its load without ever rendering it (worker_modes), so that its
#       layout can be compared while painting is not ported; that comparison is reported beside
#       the result, which stays a failure.
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: Results =================================================================================

## How a test ended.
enum Outcome:
    pass
    fail
    trapped
    crash
    timeout
    skipped

## The name of an outcome in the summary and results.tsv.
func outcome_name(outcome: Outcome) -> str:
    return match outcome:
        .pass => "pass"
        .fail => "fail"
        .trapped => "trap"
        .crash => "crash"
        .timeout => "timeout"
        .skipped => "skipped"

## What a test's run gave.
struct TestResult:
    var outcome: Outcome
    var skip: SkipReason
    ## The last stage the worker reported.
    var stage: str
    ## The trap's message, for a trap.
    var trap_message: str
    ## What differed, for a failure; how it died, for a crash.
    var detail: str
    ## A failed screenshot comparison's statistics, from the worker's log (fuzzy.lucb's
    ## messages), for results.tsv; empty otherwise.
    var statistics: str
    ## The worker's result (a Layout test's dumps, also when it stopped partway).
    var output: str
    ## The worker's standard output and error.
    var log: str
    ## Layout: whether the layout tree (the first of the three dumps) matches.
    var layout_tree_matches: bool
    ## Layout: whether the second run without rendering was made, and what it gave.
    var retried_without_rendering: bool
    var dumps_match_without_rendering: bool
    var layout_tree_matches_without_rendering: bool
    ## The second run's trap message, if it trapped.
    var trap_message_without_rendering: str

# mark: Judging a worker ========================================================================

## The text after the last "web_test: stage " line of `log`, or "start".
func last_stage(log: str) -> str:
    var stage = "start"
    var lines = strings.SplitIterator.over(log, "\n") catch failure: return stage
    while let line = lines.next():
        if strings.starts_with(line, "web_test: stage "):
            stage = (str)line.bytes["web_test: stage ".length..]
    return stage

## The message of the trap `log` reports ("trap: FILE:LINE:COLUMN: MESSAGE"), if any.
func trap_message(log: str) -> str?:
    var lines = strings.SplitIterator.over(log, "\n") catch failure: return none
    while let line = lines.next():
        if strings.starts_with(line, "trap: "):
            let location_and_message = (str)line.bytes["trap: ".length..]
            if let parts = strings.split_once(location_and_message, ": "):
                return parts.1
            return location_and_message
    return none

## The statistics of a failed screenshot comparison `log` reports (fuzzy_screenshot_match's "pixel
## error count N, with maximum error M" or its fuzzy mismatch lines), or "".
func screenshot_statistics(log: str) -> str:
    var lines = strings.SplitIterator.over(log, "\n") catch failure: return ""
    while let line = lines.next():
        if let at = strings.find(line, ": Screenshot mismatch: "):
            return (str)line.bytes[at + ": Screenshot mismatch: ".length..]
        if let at = strings.find(line, ": Fuzzy mismatch: "):
            return (str)line.bytes[at + 2..]
    return ""

## `message` with the working directory left out of every path in it: luc test runs web_test from the repository
## root, so a trap that names a file (a test, its reference) names it relative to the repository, and the notes of
## tests/expected_failures do not depend on the checkout they were written in.
func relative_to_working_directory(message: str) -> str:
    let directory = os.cwd() catch failure: return message
    return strings.replace(message, cat([directory, "/"]), "") catch failure: trap("out of memory")

## `text` without its trailing newlines (test-web compares trimmed texts).
func trim_trailing_newlines(text: str) -> str:
    var end = text.length
    while end > 0 and text.bytes[end - 1] == '\n':
        end -= 1
    return (str)text.bytes[..<end]

## The layout tree of a Layout result: what comes before the paint tree's first line.
func layout_tree_part(text: str) -> str:
    let boundaries: str[3] = ["\n\nViewportPaintable", "\n\n(no paint tree)", "\n\n(no layout tree)"]
    var end = text.length
    for boundary in boundaries:
        if let at = strings.find(text, boundary):
            if at + 1 < end:
                end = at + 1
    return trim_trailing_newlines((str)text.bytes[..<end])

## The contents of the file `path` as text, if it can be read; the text is never released.
func read_text_file(path: str) -> str?:
    let data = files.read(c_path(path)) catch failure: return none
    return (str)data.value

## Judges a Layout result against its expectation: the outcome, what differed, and whether the
## layout tree matches.
func judge_layout(test_case: const Test*, output: str) -> (Outcome, str, bool):
    let expectation_path = test_case.expectation_path else return (Outcome.fail, "no expectation", false)
    let expected = read_text_file(expectation_path) else return (Outcome.fail, "no expectation", false)
    let tree_matches = layout_tree_part(output) == layout_tree_part(expected)
    if trim_trailing_newlines(output) == trim_trailing_newlines(expected):
        return (Outcome.pass, "", true)
    return (Outcome.fail, "the layout tree differs" if not tree_matches else "the paint or stacking-context tree differs", tree_matches)

## The result of a finished worker: its exit status (none when it was stopped or died), its log
## and its result file.
func judge(test_case: const Test*, exit_code: i64?, failure: str, log: str, out_path: str) -> TestResult:
    var result = TestResult(outcome = Outcome.crash, skip = SkipReason.none_, stage = last_stage(log), trap_message = "", detail = "",
                            statistics = "", output = read_text_file(out_path) else "", log = log)
    if test_case.mode == TestMode.layout:
        let expectation_path = test_case.expectation_path else ""
        if let expected = read_text_file(expectation_path):
            result.layout_tree_matches = result.output.length > 0 and layout_tree_part(result.output) == layout_tree_part(expected)
    if let message = trap_message(log):
        result.outcome = Outcome.trapped
        result.trap_message = relative_to_working_directory(message)
        return result
    if exit_code == none:
        result.detail = failure if failure.length > 0 else "the worker died"
        return result
    let code = exit_code else trap("unreachable")
    if code != 0:
        result.detail = cat(["the worker exited with status ", format_count((u64)code)])
        return result
    match test_case.mode:
        .layout:
            let (outcome, detail, tree_matches) = judge_layout(test_case, result.output)
            result.outcome = outcome
            result.detail = detail
            result.layout_tree_matches = tree_matches
        .ref, .crash, .screenshot:
            if trim_trailing_newlines(result.output) == "pass":
                result.outcome = Outcome.pass
            else:
                result.outcome = Outcome.fail
                result.detail = "no result" if test_case.mode == TestMode.crash else "the screenshots differ"
                result.statistics = screenshot_statistics(log)
    return result

## `value` in decimal.
func format_count(value: u64) -> str:
    var buffer: u8[24]
    let text = strings.format_u64(value, buffer) catch failure: return "?"
    return strings.copy(text) catch failure: trap("out of memory")

# mark: Running workers =========================================================================

## The workers run at once by default: one per processor.
func default_jobs() -> usize:
    return (usize)os.cpus()

## A test to run: its index among the tests, and whether this is the second run, without
## rendering.
struct Job:
    var index: usize
    var no_rendering: bool

## A running worker.
struct Running:
    var job: Job
    var command: process.Command
    var started: u64
    var out_path: str

## Starts the worker of `job`. A Ref or Screenshot worker writes the PNGs of a failed comparison
## beside the test's other results (--images).
func start_job(options: const Options*, program: str, work: str, tests: const Test[], job: Job) -> Running:
    let test_case = &tests[job.index]
    let out_path = cat([work, "/", format_count((u64)job.index), ".out" if not job.no_rendering else ".retry.out"])
    let input = cat([options.root, "/", test_case.relative_path])
    var arguments: str[12] = ["--one", input, "--mode", test_mode_name(test_case.mode), "--root", options.root, "--out", out_path, "", "", "", ""]
    var count: usize = 8
    if options.gpu:
        arguments[count] = "--player"
        arguments[count + 1] = "gpu"
        count += 2
    if job.no_rendering:
        arguments[count] = "--no-rendering"
        count += 1
    elif test_case.mode == TestMode.ref or test_case.mode == TestMode.screenshot:
        arguments[count] = "--images"
        arguments[count + 1] = cat([options.results, "/", test_case.relative_path])
        count += 2
    let command = process.Command(program, arguments[..<count], output_limit = 4194304) catch failure: trap(f"web_test: cannot start a worker: {failure.message}")
    return Running(job = job, command = command, started = time.now(), out_path = out_path)

## The captured output of a finished command.
func command_output(command: const process.Command*) -> str:
    let owned = command.output() catch failure: return ""
    let text = strings.copy(owned.value) catch failure: trap("out of memory")
    owned.release()
    return text

## Why a finished command failed, if it did.
func command_failure(command: const process.Command*) -> str:
    let owned = command.error_message() catch failure: return ""
    let text = strings.copy(owned.value) catch failure: trap("out of memory")
    owned.release()
    return text

## Runs every test that is not skipped, -j at a time, and answers their results in test order.
func run_tests(options: const Options*, tests: const Test[], skips: const SkipReason[], work: str) -> ak.Vector[TestResult]:
    let program = os.executable() catch failure: trap("web_test: cannot find its own executable")
    var results: ak.Vector[TestResult]
    var queue: ak.Vector[Job]
    for index in 0..<tests.length:
        ak.vector_append_t(&results, TestResult(outcome = Outcome.skipped, skip = skips[index], stage = "", trap_message = "", detail = "", statistics = "",
                                                output = "", log = "", trap_message_without_rendering = ""))
        if skips[index] == SkipReason.none_:
            ak.vector_append_t(&queue, Job(index = index, no_rendering = false))

    let timeout = options.timeout_seconds * 1000000000
    var running: ak.Vector[Running]
    var next: usize = 0
    var finished: usize = 0
    while next < ak.vector_size(&queue) or not ak.vector_is_empty(&running):
        while next < ak.vector_size(&queue) and ak.vector_size(&running) < options.jobs:
            ak.vector_append_t(&running, start_job(options, program, work, tests, *ak.vector_at_const(&queue, next)))
            next += 1

        var slot: usize = 0
        var progressed = false
        while slot < ak.vector_size(&running):
            let worker = ak.vector_at(&running, slot)
            let timed_out = time.since(worker.started) > timeout
            if not worker.command.is_finished() and not timed_out:
                slot += 1
                continue
            progressed = true
            var done = ak.vector_take(&running, slot)
            let test_case = &tests[done.job.index]
            var result: TestResult = ---
            if not done.command.is_finished():
                done.command.cancel()
                let log = command_output(&done.command)
                done.command.close()
                result = TestResult(outcome = Outcome.timeout, skip = SkipReason.none_, stage = last_stage(log), trap_message = "",
                                    detail = cat(["no result after ", format_count(options.timeout_seconds), " s"]), statistics = "", output = "", log = log)
            else:
                let log = command_output(&done.command)
                let failure = command_failure(&done.command)
                let exit_code = done.command.exit_code()
                done.command.close()
                let status: i64? = exit_code if failure.length == 0 else none
                result = judge(test_case, status, failure, log, done.out_path)

            if done.job.no_rendering:
                let first = ak.vector_at(&results, done.job.index)
                first.retried_without_rendering = true
                first.dumps_match_without_rendering = result.outcome == Outcome.pass
                first.layout_tree_matches_without_rendering = result.layout_tree_matches
                first.trap_message_without_rendering = result.trap_message
                continue
            *ak.vector_at(&results, done.job.index) = result
            finished += 1
            if options.verbose:
                print(f"{finished}/{ak.vector_size(&queue)} {outcome_name(result.outcome)} {test_case.relative_path}")
            # It stopped before its dumps: run the dumps without rendering (see the header).
            if test_case.mode == TestMode.layout and (result.outcome == Outcome.trapped or result.outcome == Outcome.crash):
                ak.vector_append_t(&queue, Job(index = done.job.index, no_rendering = true))
        if not progressed:
            thread.sleep(2)
    return results