Luce packages
LuceLuce BaseLuciaOS

dymokomi/luc

Source of release 0.24.1

0.24.1 / src / test_programs.lucb

## `luc test`'s test programs: every directory `tests/<name>/` holding a `main.luc` or a
## `main.lucb` is a program, built with the package's dependencies and run from the package
## root, as `luc test` itself runs, with LUC_TEST_DIR naming its own directory. HOME and
## LUC_HOME point at a fresh scratch directory, removed afterwards, so no test touches the
## user's settings, libraries or keychain. It passes when it exits 0 and, when its directory
## has an `expected` file, its standard output is that file's text exactly; one that exits 0
## after printing a line `skip: reason` is skipped. The programs build and run in parallel,
## as many at once as the machine has cores, and are reported in the order of their names.
import c
import interop
import io
import memory
import os
import strings
import thread
from luce_std import files, paths, process
import packages

## What the programs are built and run with, as `luc test` was asked.
pub struct Settings:
    ## The project's root directory.
    pub var root: str
    ## The two compilers: `luce` builds a main.luc, `luce-base` a main.lucb.
    pub var luce: str
    pub var base: str
    ## `--diagnostic`: build with the diagnostic profile.
    pub var diagnostic: bool

## One program: where it lives and, once it ran, how it went.
struct Program:
    ## Its directory's name under tests/, which names it in the report.
    var name: str
    ## Its directory and its main source, full paths.
    var directory: str
    var source: str
    ## Whether it is a Luce program (main.luc) rather than a Base one.
    var luce: bool
    ## Whether it passed, or was skipped, and what to show when it did not pass: why it
    ## failed, or the reason it gave for skipping.
    var passed: bool
    var skipped: bool
    var report: str
    var done: @bool

    func release():
        strings.release(self.name)
        strings.release(self.directory)
        strings.release(self.source)
        if self.report.length > 0: strings.release(self.report)

## The programs a run works through, shared by its workers: each takes the next index.
struct Run:
    var programs: Program[]
    var settings: Settings
    var next: @u64

## How many lines of a failed program's output its report shows: the end, where it failed.
let tail_lines: usize = 20

# mark: Finding the programs ===================================================================

## The test programs under `root`/tests, in the order of their names; release them with
## `release_all`.
func find(root: str) -> Program[]!:
    var found: Program[]
    errdefer release_all(found)
    let tests = try paths.join(root, "tests")
    defer tests.release()
    if not try is_directory(tests.value): return found
    let names = try files.list((c.str)tests.value)
    defer names.release()
    for name in names.value:
        if name.length > 0 and name.bytes[0] == '.': continue
        let directory = try paths.join(tests.value, name)
        defer directory.release()
        if not try is_directory(directory.value): continue
        # a directory of test fragments (one with a TESTS) belongs to a module, and one with
        # an ORDER is a module made of files: neither is a program
        if try exists_in(directory.value, "TESTS") or try exists_in(directory.value, "ORDER"): continue
        var luce = false
        if try exists_in(directory.value, "main.luc"):
            luce = true
        elif not try exists_in(directory.value, "main.lucb"):
            continue
        let source = try paths.join(directory.value, "main.luc" if luce else "main.lucb")
        defer source.release()
        let grown = try new Program[found.length + 1] --- in memory.heap
        for i in 0..<found.length: grown[i] = found[i]
        if found.length > 0: free(found) in memory.heap
        found = grown
        found[found.length - 1] = Program(name = "", directory = "", source = "", luce = luce, passed = false, skipped = false, report = "")
        found[found.length - 1].name = try strings.copy(name)
        found[found.length - 1].directory = try strings.copy(directory.value)
        found[found.length - 1].source = try strings.copy(source.value)
    return found

## Whether the package.prisma in `directory`, if any, declares a dependency without a
## `path`, one luc fetches from the registry. A test program's manifest names little more
## than its dependencies, which the compiler reads as they are, so it is not checked as a
## package to publish: each dependency element holds at most one `path`, and fewer paths
## than dependencies leave one to fetch.
func registry_dependencies(directory: str) -> bool!:
    let manifest = try paths.join(directory, "package.prisma")
    defer manifest.release()
    if not files.exists((c.str)manifest.value): return false
    let contents = try files.read((c.str)manifest.value)
    defer contents.release()
    let text = (str)contents.value
    var dependencies: usize = 0
    var with_path: usize = 0
    var start: usize = 0
    while start < text.length:
        let end = strings.find_from(text, "\n", start) else text.length
        let line = strings.trim((str)text.bytes[start..<end])
        if strings.starts_with(line, "def dependency "): dependencies += 1
        elif strings.starts_with(line, "str path "): with_path += 1
        start = end + 1
    return with_path < dependencies

func release_all(programs: Program[]):
    for i in 0..<programs.length: programs[i].release()
    if programs.length > 0: free(programs) in memory.heap

func is_directory(path: str) -> bool!:
    let terminated = try strings.copy(path)
    defer strings.release(terminated)
    if not files.exists((c.str)terminated): return false
    return (try files.metadata((c.str)terminated)).kind == files.FileKind.directory

func exists_in(directory: str, name: str) -> bool!:
    let joined = try paths.join(directory, name)
    defer joined.release()
    return files.exists((c.str)joined.value)

## The names of the test programs, one per line, as `luc test --list` shows them.
pub func list(root: str) -> !:
    let programs = try find(root)
    defer release_all(programs)
    for program in programs: try io.stdout().write(f"tests/{program.name}\n")

# mark: Running them ===========================================================================

## How the test programs went.
pub struct Counts:
    pub var passed: u64
    pub var failed: u64
    pub var skipped: u64

## Build and run every test program, report each, and answer how many passed, failed and
## were skipped.
pub func run(settings: Settings) -> Counts!:
    var shared = Run(programs = try find(settings.root), settings = settings)
    defer release_all(shared.programs)
    var counts = Counts(passed = 0, failed = 0, skipped = 0)
    if shared.programs.length == 0: return counts
    # a program with a package.prisma of its own may declare registry dependencies the
    # package lacks; fetch them before any build, one program at a time
    for program in shared.programs:
        if try registry_dependencies(program.directory): try packages.prepare(program.directory)
    var wanted = (usize)os.cpus()
    if wanted > shared.programs.length: wanted = shared.programs.length
    if wanted == 0: wanted = 1
    let workers = try new thread.Handle[wanted] --- in memory.heap
    defer free(workers) in memory.heap
    var started: usize = 0
    while started < workers.length:
        workers[started] = thread.spawn(worker, (void*)&shared, name = "luc test") catch failure:
            _ = failure
            break
        started += 1
    # with no thread to spare, the programs run here
    if started == 0: worker((void*)&shared)
    # report in the order of the names, each as soon as it and those before it are done
    for index in 0..<shared.programs.length:
        let program = &shared.programs[index]
        while not program.done.load(.acquire): thread.sleep(20)
        if program.skipped:
            counts.skipped += 1
            try io.stdout().write(f"skip  tests/{program.name}: {program.report}\n")
        elif program.passed:
            counts.passed += 1
            try io.stdout().write(f"ok    tests/{program.name}\n")
        else:
            counts.failed += 1
            try io.stdout().write(f"FAIL  tests/{program.name}\n")
            try write_indented(program.report)
    for index in 0..<started: try workers[index].join()
    return counts

func worker(raw: void*?):
    let shared = (Run*)(raw else return)
    with memory.heap:
        while true:
            let index = (usize)shared.next.add(1, .relaxed)
            if index >= shared.programs.length: return
            let program = &shared.programs[index]
            program.passed = run_one(program, shared.settings) catch failure:
                program.report = strings.copy(failure.message) catch again:
                    _ = again
                    recover ""
                recover false
            program.done.store(true, .release)

## Build `program` into build/luc-test/<name>/ and run it from the package root with a fresh
## home there; answer whether it passed, leaving in its report what to show if not. The
## program's own scratch space, build/tests/<name>/, is made a directory (whatever stood
## there that was not one is replaced) and otherwise left to it: what luc writes lives apart,
## so the program cannot remove its own binary by clearing its scratch.
func run_one(program: Program*, settings: Settings) -> bool!:
    let build = try paths.join(settings.root, "build")
    defer build.release()
    let under = try paths.join(build.value, "luc-test")
    defer under.release()
    let output = try paths.join(under.value, program.name)
    defer output.release()
    let scratch_root = try paths.join(build.value, "tests")
    defer scratch_root.release()
    let scratch = try paths.join(scratch_root.value, program.name)
    defer scratch.release()
    let home = try paths.join(output.value, "home")
    defer home.release()
    if try is_directory(home.value): try files.remove_tree((c.str)home.value)
    for directory in [output.value, home.value, scratch.value]:
        let problem = try make_directory(directory)
        if problem.length > 0:
            program.report = problem
            return false
    let binary = try paths.join(output.value, program.name)
    defer binary.release()
    var words: str[]
    defer free_words(words)
    for word in ["build", program.source, "--native", "-o", binary.value]:
        words = try append(words, [word])
    if settings.diagnostic:
        words = try append(words, ["--profile"])
        words = try append(words, ["diagnostic"])
    let compiler = settings.luce if program.luce else settings.base
    let built = try execute(compiler, words, settings, none)
    defer built.release()
    if built.value.exit_code != 0:
        program.report = try tail("build failed", built.value.output, built.value.error_output)
        return false
    var nothing: str[]
    let ran = try execute(binary.value, nothing, settings, Scratch(home = home.value, directory = program.directory))
    defer ran.release()
    _ = files.remove_tree((c.str)home.value) catch failure:
        _ = failure
        recover ()
    let completed = ran.value
    if completed.exit_code != 0:
        var heading: u8[64]
        program.report = try tail(try strings.format(heading, f"exit status {completed.exit_code}"), completed.output, completed.error_output)
        return false
    if let reason = skip_reason(completed.output):
        program.skipped = true
        program.report = try strings.copy(reason)
        return true
    let expected = try paths.join(program.directory, "expected")
    defer expected.release()
    if files.exists((c.str)expected.value):
        let wanted = try files.read((c.str)expected.value)
        defer wanted.release()
        if (str)wanted.value != completed.output:
            program.report = try tail("its standard output differs from its expected file", completed.output, completed.error_output)
            return false
    return true

## Make `path` a directory, replacing whatever stands there that is not one (a binary an
## older layout built, say); answer an empty text, or an owned report naming the path when
## it cannot be made.
func make_directory(path: str) -> str!:
    if try is_directory(path): return ""
    let terminated = try strings.copy(path)
    defer strings.release(terminated)
    if files.exists((c.str)terminated):
        _ = files.remove_file((c.str)terminated) catch failure:
            return try strings.join(["cannot make the directory ", path, ": ", failure.message], "")
    _ = files.ensure_directory(path) catch failure:
        return try strings.join(["cannot make the directory ", path, ": ", failure.message], "")
    return ""

## What a test program runs with beyond the toolchain: its scratch home and its own
## directory.
struct Scratch:
    var home: str
    var directory: str

## The reason of the first line of `output` that starts `skip:`, if one does.
func skip_reason(output: str) -> str?:
    var start: usize = 0
    while start < output.length:
        let end = strings.find_from(output, "\n", start) else output.length
        let line = (str)output.bytes[start..<end]
        if strings.starts_with(line, "skip:"): return strings.trim((str)line.bytes[5..])
        start = end + 1
    return none

## Run `program` with `words` from the package root (luc's own directory) and capture what
## it wrote, with LUCE and LUCE_BASE naming the compilers under test, so a program that
## builds or runs Luce uses the same toolchain. A test program also gets `scratch`: HOME,
## USERPROFILE and LUC_HOME are its home, and LUC_TEST_DIR its own directory.
func execute(program: str, words: str[], settings: Settings, scratch: Scratch?) -> interop.Owned[process.Completed]!:
    var settings_texts: str[]
    defer free_words(settings_texts)
    let luce = try located(settings.luce)
    defer strings.release(luce)
    let base = try located(settings.base)
    defer strings.release(base)
    settings_texts = try append(settings_texts, ["LUCE=", luce])
    settings_texts = try append(settings_texts, ["LUCE_BASE=", base])
    if let given = scratch:
        settings_texts = try append(settings_texts, ["HOME=", given.home])
        settings_texts = try append(settings_texts, ["LUC_HOME=", given.home])
        settings_texts = try append(settings_texts, ["LUC_TEST_DIR=", given.directory])
        if os.windows: settings_texts = try append(settings_texts, ["USERPROFILE=", given.home])
    var named: str[]
    defer free_words(named)
    named = try append(named, [program])
    named = try append(named, [settings.root])
    let arguments = try new c.str[words.length] --- in memory.heap
    defer free(arguments) in memory.heap
    for i in 0..<words.length: arguments[i] = (c.str)words[i]
    let environment = try new c.str[settings_texts.length] --- in memory.heap
    defer free(environment) in memory.heap
    for i in 0..<settings_texts.length: environment[i] = (c.str)settings_texts[i]
    return try process.run((c.str)named[0], arguments, (c.str)named[1], environment)

## Where `program` is: itself when it names a path, else the first directory of PATH that
## holds it, so a test program is told the very compiler luc uses whatever its directory.
## The caller releases the answer.
func located(program: str) -> str!:
    if strings.contains(program, "/") or strings.contains(program, "\\"): return try strings.copy(program)
    let search = os.env("PATH") else return try strings.copy(program)
    let separator = ";" if os.windows else ":"
    var start: usize = 0
    while start <= search.length:
        let end = strings.find_from(search, separator, start) else search.length
        let directory = (str)search.bytes[start..<end]
        start = end + 1
        if directory.length == 0: continue
        let candidate = try paths.join(directory, program)
        defer candidate.release()
        if files.exists((c.str)candidate.value): return try strings.copy(candidate.value)
        if os.windows:
            let windows = try strings.join([candidate.value, ".exe"], "")
            if files.exists((c.str)windows): return windows
            strings.release(windows)
    return try strings.copy(program)

## `words` with one more, `pieces` joined into a NUL-terminated text it owns.
func append(words: str[], pieces: const str[]) -> str[]!:
    let word = try strings.join(pieces, "")
    errdefer strings.release(word)
    let grown = try new str[words.length + 1] --- in memory.heap
    for i in 0..<words.length: grown[i] = words[i]
    grown[words.length] = word
    if words.length > 0: free(words) in memory.heap
    return grown

func free_words(words: str[]):
    for word in words: strings.release(word)
    if words.length > 0: free(words) in memory.heap

## A failed program's report: `heading`, then the last lines of what it wrote, its
## standard output and then its standard error.
func tail(heading: str, output: str, error_output: str) -> str!:
    let combined = try strings.join([output, error_output], "")
    defer strings.release(combined)
    var start = combined.length
    var lines: usize = 0
    while start > 0:
        if combined.bytes[start - 1] == '\n' and start < combined.length:
            lines += 1
            if lines >= tail_lines: break
        start -= 1
    let shown = (str)combined.bytes[start..]
    let omitted = "...\n" if start > 0 else ""
    return try strings.join([heading, "\n", omitted, shown], "")

## `report` indented under its FAIL line, each line once, ending with a newline.
func write_indented(report: str) -> !:
    var start: usize = 0
    while start < report.length:
        let end = strings.find_from(report, "\n", start) else report.length
        try io.stdout().write(f"      {(str)report.bytes[start..<end]}\n")
        start = end + 1

test "a short report shows all of the output":
    let report = try tail("exit status 1", "a\nb\n", "c\n")
    defer strings.release(report)
    assert(report == "exit status 1\na\nb\nc\n")

test "a long report shows the last lines":
    var lines: str[]
    defer free_words(lines)
    for i in 0..<30:
        var line: u8[16]
        lines = try append(lines, [try strings.format(line, f"{i}\n")])
    let output = try strings.join(lines, "")
    defer strings.release(output)
    let report = try tail("exit status 2", output, "")
    defer strings.release(report)
    assert(strings.starts_with(report, "exit status 2\n...\n10\n11\n"))
    assert(strings.ends_with(report, "\n29\n"))

test "only a dependency without a path is fetched":
    let directory = try files.create_temporary_directory("/tmp")
    defer _ = files.remove_tree((c.str)directory.value) catch: recover ()
    defer directory.release()
    assert(not try registry_dependencies(directory.value))
    let manifest = try paths.join(directory.value, "package.prisma")
    defer manifest.release()
    try files.write((c.str)manifest.value, b"#prisma 4.0\ndef package \"t\" {\n    def dependency \"a\" {\n        str path = \"../a\"\n    }\n}\n")
    assert(not try registry_dependencies(directory.value))
    try files.write((c.str)manifest.value, b"#prisma 4.0\ndef package \"t\" {\n    def dependency \"a\" {\n        str path = \"../a\"\n    }\n    def dependency \"b\" {\n        str owner = \"acme\"\n    }\n}\n")
    assert(try registry_dependencies(directory.value))

test "a program named by its path is not looked for":
    let found = try located("/opt/luce/bin/luce-base")
    defer strings.release(found)
    assert(found == "/opt/luce/bin/luce-base")

test "a line starting skip: skips, with its reason":
    let reason = skip_reason("checking\nskip: no Vulkan device\n") else error(0, "no reason")
    assert(reason == "no Vulkan device")
    assert(skip_reason("ok\nnot skip: here\n") == none)