Luce packages
LuceLuce BaseLuciaOS

dymokomi/luc

Source of release 0.24.3

0.24.3 / src / package_tests.lucb

## `luc test` and `luc check`. `luc test`: the `test` blocks of every module of the package,
## each run by the compiler of
## its language, and the package's test programs (test_programs), which build and run on
## threads of their own while the test blocks run. The Luce modules go to `luce test
## --package` and the Base modules to `luce-base test --package`, whatever the package's
## language, so a mixed package runs both halves. Each run starts from the package's entry
## when it is in that language and otherwise from its first module, since `--package` takes
## every module under the source root either way. A run that fails without reporting a failed
## test (a build error in the sources, a crash) counts as one failure, named in the report. A
## package with no test at all is refused: every package has tests. `luc check` checks the
## same modules from the same starts without building, the Base ones for every target.
import c
import interop
import io
import memory
import os
import strings
from luce_std import files, paths, process
import test_programs

## What `luc test` was asked to test, as main loaded it.
pub struct Request:
    ## The project's root directory.
    pub var root: str
    ## The source root under it, `src` unless the manifest says otherwise.
    pub var source: str
    ## The entry's path, or empty for a library with no starter module.
    pub var entry: str
    ## The two compilers.
    pub var luce: str
    pub var base: str
    ## `--diagnostic`: the diagnostic profile, which builds the tests.
    pub var diagnostic: bool
    ## What followed `--`, given to every compiler run.
    pub var passthrough: str[]
    ## `--list`: name the tests instead of running them.
    pub var list: bool
    ## `--jobs`: how many test programs build and run at once; zero for the default.
    pub var jobs: usize

## The modules a run can start from and the tests the sources declare: the first Luce
## module and the first Base module under the source root, a `.lucb` file or a directory
## module (one with an ORDER), as full paths the caller releases; empty when there is none.
struct Sources:
    var luce: str
    var base: str
    var declared: u64

    func release():
        if self.luce.length > 0: strings.release(self.luce)
        if self.base.length > 0: strings.release(self.base)

## One compiler's run: its status and the counts its report ended with.
struct Outcome:
    var status: i32
    var passed: u64
    var failed: u64

## Test the package and answer the exit status: the worst of the runs; 1 when a run failed
## without reporting a failed test, when the sources declare tests and no run reported any,
## or when the package has no test at all.
##
## The test programs start first and build and run beside the test blocks: every compile
## writes the build cache by renaming a finished file into place, and each program builds
## into a directory of its own, so the two cannot meet. While programs run beside them, the
## test-block runs get the same share of the processors as each program's build.
pub func run(request: Request) -> i32!:
    if request.list: return try list(request)
    var sources = try scan(request.root, request.source)
    defer sources.release()
    let jobs = request.jobs if request.jobs > 0 else test_programs.default_jobs()
    var programs_run = try test_programs.prepare(test_programs.Settings(root = request.root,
        luce = request.luce, base = request.base, diagnostic = request.diagnostic, jobs = jobs))
    errdefer test_programs.stop(&programs_run)
    test_programs.launch(&programs_run)
    let share = test_programs.compile_share(&programs_run)
    # each compiler starts from the entry when it is in its language, else its first module
    let luce_start = request.entry if strings.ends_with(request.entry, ".luc") else sources.luce
    let base_start = request.entry if strings.ends_with(request.entry, ".lucb") else sources.base
    var total = Outcome(status = 0, passed = 0, failed = 0)
    var runs: usize = 0
    # runs that failed without reporting a failed test, each counted as one failure
    var broken: u64 = 0
    for start in [luce_start, base_start]:
        if start.length == 0: continue
        let luce = strings.ends_with(start, ".luc")
        let compiler = request.luce if luce else request.base
        let outcome = try test_with(compiler, start, request, share)
        if outcome.status != 0 and total.status == 0: total.status = outcome.status
        total.passed += outcome.passed
        total.failed += outcome.failed
        runs += 1
        if outcome.status != 0 and outcome.failed == 0:
            broken += 1
            let half = "Luce" if luce else "Base"
            let named = "luce" if luce else "luce-base"
            try io.stdout().write(f"FAIL  the {half} test blocks: {named} test exited with status {outcome.status} and reported no failed test\n")
    # a report of no tests from sources that declare some means a run missed them
    if total.status == 0 and total.passed + total.failed == 0 and sources.declared > 0:
        broken += 1
        total.status = 1
        try io.stdout().write(f"FAIL  the test blocks: the sources declare {sources.declared} test(s) but none ran\n")
    let blocks = total.passed + total.failed
    let programs = try test_programs.finish(&programs_run)
    let program_count = programs.passed + programs.failed + programs.skipped
    if programs.failed > 0 and total.status == 0: total.status = 1
    total.passed += programs.passed
    total.failed += programs.failed + broken
    if blocks + program_count == 0 and total.status == 0:
        try io.stderr().write("luc test: no tests found: add a `test` block or a tests/<name>/main program\n")
        return 1
    var skipped_text: u8[48]
    let skipped = try strings.format(skipped_text, f", {programs.skipped} skipped") if programs.skipped > 0 else ""
    var broken_text: u8[64]
    let run_word = "test-block run" if broken == 1 else "test-block runs"
    let broken_runs = try strings.format(broken_text, f", {broken} failed {run_word}") if broken > 0 else ""
    # one total when more than one run contributed
    if runs > 1 or (runs > 0 and program_count > 0):
        let block_word = "test block" if blocks == 1 else "test blocks"
        let program_word = "program" if program_count == 1 else "programs"
        try io.stdout().write(f"total: {total.passed} passed, {total.failed} failed{skipped} ({blocks} {block_word}{broken_runs}, {program_count} {program_word})\n")
    elif program_count > 0:
        try io.stdout().write(f"{programs.passed} passed\n")
        if programs.failed > 0: try io.stdout().write(f"{programs.failed} failed\n")
        if programs.skipped > 0: try io.stdout().write(f"{programs.skipped} skipped\n")
    return total.status

## The targets a check of the Base modules sees in turn (base.md ยง19.5), besides the host:
## each decides the platform constants its own way, so a branch for one target is checked
## under that target. One per system; the host's own system is the plain check.
let other_targets = ["arm64-macos", "x86_64-linux", "x86_64-windows"]

## `luc check`: every module of the package under `root`/`source`, imported by the entry or
## not, type-checked without building: the Luce modules by `luce check --package` and the
## Base modules by `luce-base check --package`, for the host and then for each other system's
## target, each from the entry when it is in that language and otherwise from its first
## module, as `run` starts. The first failing check ends it, since every target would mostly
## report the same errors again. `passthrough` goes to every run (`-W`, say). The exit status
## is the failed run's; a package with no module is refused.
pub func check(root: str, source: str, entry: str, luce: str, base: str, passthrough: str[]) -> i32!:
    var sources = try scan(root, source)
    defer sources.release()
    let luce_start = entry if strings.ends_with(entry, ".luc") else sources.luce
    let base_start = entry if strings.ends_with(entry, ".lucb") else sources.base
    if luce_start.length == 0 and base_start.length == 0:
        try io.stderr().write(f"luc check: no module to check under {source}\n")
        return 1
    if luce_start.length > 0:
        let status = try check_with(luce, luce_start, "", passthrough)
        if status != 0: return status
    if base_start.length > 0:
        let host = "-macos" if os.macos else ("-linux" if os.linux else "-windows")
        var status = try check_with(base, base_start, "", passthrough)
        for target in other_targets:
            if status != 0: return status
            if not strings.ends_with(target, host):
                status = try check_with(base, base_start, target, passthrough)
        return status
    return 0

## Run `compiler check start --package`, for `target` when there is one, show what it wrote
## and answer its status.
func check_with(compiler: str, start: str, target: str, passthrough: str[]) -> i32!:
    var words: str[5] = ---
    var count: usize = 0
    for word in ["check", start, "--package"]:
        words[count] = word
        count += 1
    if target.length > 0:
        words[count] = "--target"
        words[count + 1] = target
        count += 2
    let argv = try new c.str[count + passthrough.length] ---
    var made: usize = 0
    defer release_arguments(argv, made)
    for i in 0..<count:
        argv[made] = (c.str)try strings.copy(words[i])
        made += 1
    for word in passthrough:
        argv[made] = (c.str)try strings.copy(word)
        made += 1
    let program = try strings.copy(compiler)
    defer strings.release(program)
    let ran = try process.run((c.str)program, argv)
    defer ran.release()
    if ran.value.output.length > 0: try io.stdout().write(ran.value.output)
    if ran.value.error_output.length > 0:
        # another target's failure is named, since the host's passed
        if target.length > 0 and ran.value.exit_code != 0: try io.stderr().write(f"luc check: for {target}:\n")
        try io.stderr().write(ran.value.error_output)
    return ran.value.exit_code

## `luc test --list`: the test programs, then the test blocks the sources declare, one per
## line, without building anything.
func list(request: Request) -> i32!:
    try test_programs.list(request.root)
    let joined = try paths.join(request.root, request.source)
    defer joined.release()
    if not files.exists((c.str)joined.value): return 0
    let listing = try files.walk((c.str)joined.value)
    defer listing.release()
    for relative in listing.value:
        if hidden(relative) or try in_nested_package(joined.value, relative): continue
        if not strings.ends_with(relative, ".luc") and not strings.ends_with(relative, ".lucb"): continue
        let path = try paths.join(joined.value, relative)
        defer path.release()
        _ = try visit_tests(path.value, true)
    return 0

## Run `compiler test start --package` with the request's profile and arguments, show what
## it wrote, and read the counts from its report. A `share` above zero is the LUCE_BASE_JOBS
## it runs with, beside the test programs.
func test_with(compiler: str, start: str, request: Request, share: usize) -> Outcome!:
    var words: str[8] = ---
    var count: usize = 0
    for word in ["test", start, "--package"]:
        words[count] = word
        count += 1
    if request.diagnostic:
        words[count] = "--profile"
        words[count + 1] = "diagnostic"
        count += 2
    let argv = try new c.str[count + request.passthrough.length] ---
    var made: usize = 0
    defer release_arguments(argv, made)
    for i in 0..<count:
        argv[made] = (c.str)try strings.copy(words[i])
        made += 1
    for word in request.passthrough:
        argv[made] = (c.str)try strings.copy(word)
        made += 1
    let program = try strings.copy(compiler)
    defer strings.release(program)
    let ran = try run_sharing(program, argv, share)
    defer ran.release()
    var outcome = Outcome(status = ran.value.exit_code, passed = 0, failed = 0)
    read_counts(ran.value.output, &outcome)
    # a run that found no test block (the Base half of a Luce package with none, say) has
    # nothing to show but its `0 passed`: luc's total speaks for it
    let empty = outcome.status == 0 and outcome.passed + outcome.failed == 0
    if ran.value.output.length > 0 and not empty: try io.stdout().write(ran.value.output)
    if ran.value.error_output.length > 0: try io.stderr().write(ran.value.error_output)
    return outcome

## `process.run`, with LUCE_BASE_JOBS set to `share` when it is above zero.
func run_sharing(program: str, arguments: c.str[], share: usize) -> interop.Owned[process.Completed]!:
    if share == 0: return try process.run((c.str)program, arguments)
    var number: u8[24]
    let setting = try strings.join(["LUCE_BASE_JOBS=", try strings.format(number, f"{share}")], "")
    defer strings.release(setting)
    let environment = try new c.str[1] --- in memory.heap
    defer free(environment) in memory.heap
    environment[0] = (c.str)setting
    return try process.run((c.str)program, arguments, "", environment)

func release_arguments(arguments: c.str[], count: usize):
    for i in 0..<count: strings.release((str)arguments[i])
    if arguments.length > 0: free(arguments) in memory.heap

## The report's closing counts, the lines `N passed` and `N failed`; the last of each wins.
func read_counts(output: str, outcome: Outcome*):
    var start: usize = 0
    while start < output.length:
        let end = strings.find_from(output, "\n", start) else output.length
        let line = (str)output.bytes[start..<end]
        start = end + 1
        let (number, word) = strings.split_once(line, " ") else continue
        let value = strings.parse_u64(number) else continue
        if word == "passed": outcome.passed = value
        elif word == "failed": outcome.failed = value

# mark: The sources ============================================================================

## The package's modules under `root`/`source`, in the order of their paths, and how many
## tests they declare. A hidden directory and one holding a package of its own are left out,
## as the compilers leave them out of `--package`.
func scan(root: str, source: str) -> Sources!:
    var sources = Sources(luce = "", base = "", declared = 0)
    errdefer sources.release()
    let joined = try paths.join(root, source)
    defer joined.release()
    let directory = try strings.copy(joined.value)
    defer strings.release(directory)
    if not files.exists((c.str)directory): return sources
    let listing = try files.walk((c.str)directory)
    defer listing.release()
    for relative in listing.value:
        if hidden(relative) or try in_nested_package(directory, relative): continue
        let path = try paths.join(directory, relative)
        defer path.release()
        let luce_module = strings.ends_with(relative, ".luc")
        let base_file = strings.ends_with(relative, ".lucb")
        if luce_module or base_file:
            sources.declared += try declared_tests(path.value)
        if luce_module and sources.luce.length == 0:
            sources.luce = try strings.copy(path.value)
        # the walk lists a directory before its files, so a directory module comes first
        let base_module = base_file or try has_file(path.value, "ORDER")
        if base_module and sources.base.length == 0 and not try in_ancestor(directory, relative, "ORDER"):
            sources.base = try strings.copy(path.value)
    return sources

## Whether a part of `relative` starts with a dot.
func hidden(relative: str) -> bool:
    if relative.length > 0 and relative.bytes[0] == '.': return true
    return strings.contains(relative, "/.")

## Whether a directory between `directory` and the entry `relative` has a package.prisma:
## that package's modules are its own.
func in_nested_package(directory: str, relative: str) -> bool!:
    return try in_ancestor(directory, relative, "package.prisma")

## Whether a directory between `directory` and the entry `relative`, not counting `relative`
## itself, holds a file `name`: an ORDER makes everything below it part of one module.
func in_ancestor(directory: str, relative: str, name: str) -> bool!:
    for i in 0..<relative.length:
        if relative.bytes[i] != '/': continue
        let below = try paths.join(directory, (str)relative.bytes[..<i])
        defer below.release()
        if try has_file(below.value, name): return true
    return false

func has_file(directory: str, name: str) -> bool!:
    let joined = try paths.join(directory, name)
    defer joined.release()
    let terminated = try strings.copy(joined.value)
    defer strings.release(terminated)
    return files.exists((c.str)terminated)

## How many top-level `test "..."` declarations the source at `path` holds.
func declared_tests(path: str) -> u64!:
    return try visit_tests(path, false)

## The top-level `test "..."` declarations of the source at `path`, counted, and with
## `show` written out by name, one per line.
func visit_tests(path: str, show: bool) -> u64!:
    let terminated = try strings.copy(path)
    defer strings.release(terminated)
    let contents = try files.read((c.str)terminated)
    defer contents.release()
    let text = (str)contents.value
    var count: u64 = 0
    var start: usize = 0
    while start < text.length:
        let end = strings.find_from(text, "\n", start) else text.length
        let line = (str)text.bytes[start..<end]
        if strings.starts_with(line, "test \""):
            count += 1
            if show:
                let named = (str)line.bytes[6..]
                let close = strings.find_last(named, "\"") else named.length
                try io.stdout().write(f"{(str)named.bytes[..<close]}\n")
        start = end + 1
    return count

test "the last counts of a report win":
    var outcome = Outcome(status = 0, passed = 0, failed = 0)
    read_counts("ok    a\n1 passed\nFAIL  b\n2 passed\n1 failed\n", &outcome)
    assert(outcome.passed == 2 and outcome.failed == 1)

test "a hidden path has a part starting with a dot":
    assert(hidden(".git/x.lucb") and hidden("a/.cache/b.lucb") and not hidden("a/b.lucb"))