Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.1

0.6.1 / tools / site_sweep / driver.lucb

#==============================================================================================
#
#   driver - runs a worker per site, a few at a time, and summarizes the outcomes
#
#   DESCRIPTION:
#       Each worker is this program with `--one URL`, run through luce-std's process.Command,
#       which captures its output. A worker that prints a result line is judged by it; one that
#       trapped by its trap line (and the stage it reached); one that ran out of time is
#       stopped. The summary lists every site in order, then the traps grouped by message,
#       the most frequent first.
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: Sites ===================================================================================

## The sites to visit: the URLs on the command line, else the lines of the sites file.
func driver_sites(options: const Options*) -> ak.Vector[str]:
    if not ak.vector_is_empty(&options.urls):
        return options.urls
    var sites: ak.Vector[str]
    let data = files.read(c_path(options.sites)) catch failure: trap(f"site_sweep: cannot read {options.sites}: {failure.message}")
    var lines = strings.SplitIterator.over((str)data.value, "\n") catch failure: trap("out of memory")
    while let line = lines.next():
        var text = line
        if let comment = strings.find(text, "#"):
            text = (str)text.bytes[..<comment]
        text = strings.trim(text)
        if text.length > 0:
            ak.vector_append_t(&sites, text)
    return sites

# mark: Workers =================================================================================

## A running worker.
struct Running:
    var index: usize
    var command: process.Command
    var started: u64

## How a site went.
struct SiteResult:
    var outcome: str
    var detail: str

## The value after `prefix` on the last line of `log` that starts with it, if any.
func last_line_value(log: str, prefix: str) -> str?:
    var value: str? = none
    var lines = strings.SplitIterator.over(log, "\n") catch failure: return none
    while let line = lines.next():
        if strings.starts_with(line, prefix):
            value = (str)line.bytes[prefix.length..]
    return value

## The message of the trap `log` reports ("trap: FILE:LINE:COLUMN: MESSAGE"), if any.
func trap_message(log: str) -> str?:
    let location_and_message = last_line_value(log, "trap: ") else return none
    if let parts = strings.split_once(location_and_message, ": "):
        return parts.1
    return location_and_message

## The captured output of a finished command.
func command_output(command: const process.Command*) -> str:
    let owned = command.output() catch failure: return ""
    let text = strings.copy(owned.value) catch failure: trap("out of memory")
    owned.release()
    return text

## How a finished worker went, from its output.
func judge(log: str, exit_code: i64?) -> SiteResult:
    let stage = last_line_value(log, "site_sweep: stage ") else "start"
    if let message = trap_message(log):
        return SiteResult(outcome = "trap", detail = cat([message, " (stage ", stage, ")"]))
    if let result = last_line_value(log, "site_sweep: result "):
        if let parts = strings.split_once(result, " "):
            return SiteResult(outcome = parts.0, detail = parts.1)
        return SiteResult(outcome = result, detail = "")
    if let code = exit_code:
        return SiteResult(outcome = "crash", detail = cat(["exit status ", decimal((u64)code), " at stage ", stage]))
    return SiteResult(outcome = "crash", detail = cat(["died at stage ", stage]))

## Runs a worker per site, `options.jobs` at a time, and answers their results in site order.
func run_sites(options: const Options*, sites: const ak.Vector[str]*) -> ak.Vector[SiteResult]:
    let program = os.executable() catch failure: trap("site_sweep: cannot find its own executable")
    var results: ak.Vector[SiteResult]
    for index in 0..<ak.vector_size(sites):
        ak.vector_append_t(&results, SiteResult(outcome = "", detail = ""))
    let timeout = options.timeout_seconds * 1000000000
    var running: ak.Vector[Running]
    var next: usize = 0
    while next < ak.vector_size(sites) or not ak.vector_is_empty(&running):
        while next < ak.vector_size(sites) and ak.vector_size(&running) < options.jobs:
            var arguments: str[5] = ["--one", *ak.vector_at_const(sites, next), "--resources", options.resources, "--watch"]
            let count: usize = 5 if options.watch else 4
            let command = process.Command(program, arguments[..<count], output_limit = 4194304) catch failure: trap(f"site_sweep: cannot start a worker: {failure.message}")
            ak.vector_append_t(&running, Running(index = next, command = command, started = time.now()))
            next += 1
        var slot: usize = 0
        var progressed = false
        while slot < ak.vector_size(&running):
            let worker = ak.vector_at(&running, slot)
            let timed_out = time.since(worker.started) > timeout
            if not worker.command.is_finished() and not timed_out:
                slot += 1
                continue
            progressed = true
            var done = ak.vector_take(&running, slot)
            var result: SiteResult = ---
            if not done.command.is_finished():
                done.command.cancel()
                let log = command_output(&done.command)
                let stage = last_line_value(log, "site_sweep: stage ") else "start"
                result = SiteResult(outcome = "timeout", detail = cat(["stopped after ", decimal(options.timeout_seconds), " s at stage ", stage]))
            else:
                let log = command_output(&done.command)
                result = judge(log, done.command.exit_code())
            done.command.close()
            let site = *ak.vector_at_const(sites, done.index)
            print(f"{site}: {result.outcome} {result.detail}")
            *ak.vector_at(&results, done.index) = result
        if not progressed:
            thread.sleep(20)
    return results

# mark: The summary =============================================================================

## A trap message and how many sites hit it.
struct TrapCount:
    var message: str
    var count: usize

## Prints every site's outcome, then the traps by count.
func print_summary(sites: const ak.Vector[str]*, results: const ak.Vector[SiteResult]*):
    print("")
    print("== sites")
    var traps: ak.Vector[TrapCount]
    var outcome_counts: ak.Vector[TrapCount]
    for index in 0..<ak.vector_size(sites):
        let site = *ak.vector_at_const(sites, index)
        let result = ak.vector_at_const(results, index)
        print(f"{site} | {result.outcome} | {result.detail}")
        count_key(&outcome_counts, result.outcome)
        if result.outcome == "trap":
            var message = result.detail
            if let parts = strings.split_once(result.detail, " (stage "):
                message = parts.0
            count_key(&traps, message)
    print("")
    print("== outcomes")
    sort_counts(&traps)
    sort_counts(&outcome_counts)
    for entry in ak.vector_span(&outcome_counts):
        print(f"{entry.count} {entry.message}")
    print("")
    print("== traps by count")
    for entry in ak.vector_span(&traps):
        print(f"{entry.count} {entry.message}")

## Counts one more `key` in `counts`.
func count_key(counts: ak.Vector[TrapCount]*, key: str):
    for entry in &ak.vector_span(counts):
        if entry.message == key:
            entry.count += 1
            return
    ak.vector_append_t(counts, TrapCount(message = key, count = 1))

## Sorts `counts` by count, the largest first, ties in first-seen order (an insertion sort: the
## lists are short).
func sort_counts(counts: ak.Vector[TrapCount]*):
    let entries = ak.vector_span(counts)
    for index in 1..<entries.length:
        let entry = entries[index]
        var at = index
        while at > 0 and entries[at - 1].count < entry.count:
            entries[at] = entries[at - 1]
            at -= 1
        entries[at] = entry

## Visits every site and prints the summary. Answers 0: the sweep is a survey, not a test.
func driver_main(options: const Options*) -> i32:
    let sites = driver_sites(options)
    print(f"site_sweep: {ak.vector_size(&sites)} sites, {options.jobs} at a time")
    let results = run_sites(options, &sites)
    print_summary(&sites, &results)
    return 0