dymokomi/luc
Source of release 0.24.2
0.24.2 / src / test_programs.lucb
## `luc test`'s test programs: every directory `tests/<name>/` holding a `main.luc` or a
## `main.lucb` is a program, built with the package's dependencies and run from the package
## root, as `luc test` itself runs, with LUC_TEST_DIR naming its own directory. HOME and
## LUC_HOME point at a fresh scratch directory, removed afterwards, so no test touches the
## user's settings, libraries or keychain. It passes when it exits 0 and, when its directory
## has an `expected` file, its standard output is that file's text exactly; one that exits 0
## after printing a line `skip: reason` is skipped.
##
## The programs build and run in parallel, `--jobs` at a time (half the processors unless
## told), and are reported in the order of their names, each with all it has to show. Each
## compile is itself parallel inside the compiler, so a build or a run of a program gets
## LUCE_BASE_JOBS set to its share of the processors, unless the environment sets it. A
## program whose directory holds a file named `serial` must run alone: it is built with the
## others, and run after them, one at a time, once nothing else is running.
import c
import interop
import io
import memory
import os
import strings
import thread
from luce_std import files, paths, process
import packages
## What the programs are built and run with, as `luc test` was asked.
pub struct Settings:
## The project's root directory.
pub var root: str
## The two compilers: `luce` builds a main.luc, `luce-base` a main.lucb.
pub var luce: str
pub var base: str
## `--diagnostic`: build with the diagnostic profile.
pub var diagnostic: bool
## `--jobs`: how many programs build and run at once, at least one.
pub var jobs: usize
## One program: where it lives and, once it ran, how it went.
struct Program:
## Its directory's name under tests/, which names it in the report.
var name: str
## Its directory and its main source, full paths.
var directory: str
var source: str
## Whether it is a Luce program (main.luc) rather than a Base one.
var luce: bool
## Whether it must run alone, its directory holding a file named `serial`.
var serial: bool
## Whether it passed, or was skipped, and what to show when it did not pass: why it
## failed, or the reason it gave for skipping.
var passed: bool
var skipped: bool
var report: str
var done: @bool
func release():
strings.release(self.name)
strings.release(self.directory)
strings.release(self.source)
if self.report.length > 0: strings.release(self.report)
## The programs a run works through, shared by its workers: each takes the next index,
## builds that program and, unless it is serial, runs it. The worker that finishes the last
## of them runs the serial programs, once the caller runs nothing else beside them.
pub struct Run:
var programs: Program[]
var settings: Settings
var next: @u64
## How many programs the workers are through with.
var finished: @u64
## Set when the caller runs nothing more beside the programs (`finish`).
var alone: @bool
var workers: thread.Handle[]
var started: usize
## The LUCE_BASE_JOBS each compile beside others gets (`compile_jobs`), or zero.
var share: usize
## How many lines of a failed program's output its report shows: the end, where it failed.
let tail_lines: usize = 20
# mark: Finding the programs ===================================================================
## The test programs under `root`/tests, in the order of their names; release them with
## `release_all`.
func find(root: str) -> Program[]!:
var found: Program[]
errdefer release_all(found)
let tests = try paths.join(root, "tests")
defer tests.release()
if not try is_directory(tests.value): return found
let names = try files.list((c.str)tests.value)
defer names.release()
for name in names.value:
if name.length > 0 and name.bytes[0] == '.': continue
let directory = try paths.join(tests.value, name)
defer directory.release()
if not try is_directory(directory.value): continue
# a directory of test fragments (one with a TESTS) belongs to a module, and one with
# an ORDER is a module made of files: neither is a program
if try exists_in(directory.value, "TESTS") or try exists_in(directory.value, "ORDER"): continue
var luce = false
if try exists_in(directory.value, "main.luc"):
luce = true
elif not try exists_in(directory.value, "main.lucb"):
continue
let source = try paths.join(directory.value, "main.luc" if luce else "main.lucb")
defer source.release()
let grown = try new Program[found.length + 1] --- in memory.heap
for i in 0..<found.length: grown[i] = found[i]
if found.length > 0: free(found) in memory.heap
found = grown
found[found.length - 1] = Program(name = "", directory = "", source = "", luce = luce,
serial = try exists_in(directory.value, "serial"), passed = false, skipped = false, report = "")
found[found.length - 1].name = try strings.copy(name)
found[found.length - 1].directory = try strings.copy(directory.value)
found[found.length - 1].source = try strings.copy(source.value)
return found
## Whether the package.prisma in `directory`, if any, declares a dependency without a
## `path`, one luc fetches from the registry. A test program's manifest names little more
## than its dependencies, which the compiler reads as they are, so it is not checked as a
## package to publish: each dependency element holds at most one `path`, and fewer paths
## than dependencies leave one to fetch.
func registry_dependencies(directory: str) -> bool!:
let manifest = try paths.join(directory, "package.prisma")
defer manifest.release()
if not files.exists((c.str)manifest.value): return false
let contents = try files.read((c.str)manifest.value)
defer contents.release()
let text = (str)contents.value
var dependencies: usize = 0
var with_path: usize = 0
var start: usize = 0
while start < text.length:
let end = strings.find_from(text, "\n", start) else text.length
let line = strings.trim((str)text.bytes[start..<end])
if strings.starts_with(line, "def dependency "): dependencies += 1
elif strings.starts_with(line, "str path "): with_path += 1
start = end + 1
return with_path < dependencies
func release_all(programs: Program[]):
for i in 0..<programs.length: programs[i].release()
if programs.length > 0: free(programs) in memory.heap
func is_directory(path: str) -> bool!:
let terminated = try strings.copy(path)
defer strings.release(terminated)
if not files.exists((c.str)terminated): return false
return (try files.metadata((c.str)terminated)).kind == files.FileKind.directory
func exists_in(directory: str, name: str) -> bool!:
let joined = try paths.join(directory, name)
defer joined.release()
return files.exists((c.str)joined.value)
## The names of the test programs, one per line, as `luc test --list` shows them.
pub func list(root: str) -> !:
let programs = try find(root)
defer release_all(programs)
for program in programs:
let note = " (serial)" if program.serial else ""
try io.stdout().write(f"tests/{program.name}{note}\n")
# mark: Running them ===========================================================================
## How the test programs went.
pub struct Counts:
pub var passed: u64
pub var failed: u64
pub var skipped: u64
## How many programs run at once unless `--jobs` says: half the processors, at least one.
## A program's build is parallel inside the compiler too, so with half as many builds as
## processors each has two threads of its own (`compile_jobs`), rather than every build
## contending for every processor.
pub func default_jobs() -> usize:
let half = (usize)os.cpus() // 2
return half if half > 0 else 1
## The threads each of `jobs` compiles running at once may use, its share of the
## processors: the LUCE_BASE_JOBS it is given. Zero when the environment sets
## LUCE_BASE_JOBS, which is then left as it is. Four programs on sixteen processors build
## with four threads each, whatever `--jobs` allowed.
func compile_jobs(jobs: usize) -> usize:
if os.env("LUCE_BASE_JOBS") != none: return 0
let share = (usize)os.cpus() // (jobs if jobs > 0 else 1)
return share if share > 0 else 1
## The test programs under the package, found, with the registry dependencies they declare
## fetched, ready to `launch`; `finish` or `stop` releases them.
pub func prepare(settings: Settings) -> Run!:
let programs = try find(settings.root)
errdefer release_all(programs)
# a program with a package.prisma of its own may declare registry dependencies the
# package lacks; fetch them before any build, one program at a time
for program in programs:
if try registry_dependencies(program.directory): try packages.prepare(program.directory)
var wanted = settings.jobs if settings.jobs > 0 else 1
if wanted > programs.length: wanted = programs.length
var workers: thread.Handle[]
if wanted > 0: workers = try new thread.Handle[wanted] --- in memory.heap
return Run(programs = programs, settings = settings, workers = workers, started = 0,
share = compile_jobs(wanted) if wanted > 0 else 0)
## The LUCE_BASE_JOBS a compile running beside the programs gets: each worker's share of
## the processors, or zero when there are no programs or the environment sets it.
pub func compile_share(run: Run*) -> usize:
return run.share
## Start building and running the programs, on as many threads as `--jobs` asks; the caller
## may run its own work beside them until `finish`. With no thread to spare, `finish` runs
## them itself.
pub func launch(run: Run*):
while run.started < run.workers.length:
run.workers[run.started] = thread.spawn(worker, (void*)run, name = "luc test") catch failure:
_ = failure
break
run.started += 1
## Report every program in the order of their names, each as soon as it and those before it
## are done, release them, and answer how many passed, failed and were skipped. The caller
## runs nothing more beside the programs from here, so the serial ones may start.
pub func finish(run: Run*) -> Counts!:
defer stop(run)
var counts = Counts(passed = 0, failed = 0, skipped = 0)
run.alone.store(true, .release)
if run.started == 0 and run.programs.length > 0: worker((void*)run)
for index in 0..<run.programs.length:
let program = &run.programs[index]
while not program.done.load(.acquire): thread.sleep(20)
if program.skipped:
counts.skipped += 1
try io.stdout().write(f"skip tests/{program.name}: {program.report}\n")
elif program.passed:
counts.passed += 1
try io.stdout().write(f"ok tests/{program.name}\n")
else:
counts.failed += 1
try io.stdout().write(f"FAIL tests/{program.name}\n")
try write_indented(program.report)
return counts
## Wait for the workers and release the programs: at the end of `finish`, or on the way out
## when the caller fails between `launch` and `finish`. A second call does nothing.
pub func stop(run: Run*):
run.alone.store(true, .release)
for index in 0..<run.started:
_ = run.workers[index].join() catch failure:
_ = failure
recover ()
run.started = 0
release_all(run.programs)
if run.workers.length > 0: free(run.workers) in memory.heap
var no_programs: Program[]
var no_workers: thread.Handle[]
run.programs = no_programs
run.workers = no_workers
## A worker: take the next program until none is left, building each and running it unless
## it is serial. The worker that finishes the last program runs the serial ones.
func worker(raw: void*?):
let shared = (Run*)(raw else return)
with memory.heap:
while true:
let index = (usize)shared.next.add(1, .relaxed)
if index >= shared.programs.length: return
let program = &shared.programs[index]
program.passed = attempt(program, shared, true, not program.serial)
# a serial program that built waits for its turn
if not program.serial or not program.passed: program.done.store(true, .release)
if (usize)shared.finished.add(1, .acq_rel) + 1 == shared.programs.length: run_serial(shared)
## Run the serial programs one at a time, once every other program is through and the
## caller runs nothing beside them, so each runs alone, with every processor to itself.
func run_serial(shared: Run*):
while not shared.alone.load(.acquire): thread.sleep(20)
for index in 0..<shared.programs.length:
let program = &shared.programs[index]
if not program.serial or program.done.load(.acquire): continue
program.passed = attempt(program, shared, false, true)
program.done.store(true, .release)
## `run_one`, a failure of luc's own (a directory it cannot list, say) failing the program
## with its message.
func attempt(program: Program*, shared: Run*, build: bool, run: bool) -> bool:
let passed = run_one(program, shared.settings, shared.share, build, run) catch failure:
program.report = strings.copy(failure.message) catch again:
_ = again
recover ""
recover false
return passed
## With `build`, build `program` into build/luc-test/<name>/; with `run`, run it from the
## package root with a fresh home there. Answer whether it passed (or, built only, whether it
## built), leaving in its report what to show if not. The program's own scratch space,
## build/tests/<name>/, is made a directory (whatever stood there that was not one is
## replaced) and otherwise left to it: what luc writes lives apart, so the program cannot
## remove its own binary by clearing its scratch.
func run_one(program: Program*, settings: Settings, share: usize, build: bool, run: bool) -> bool!:
let built_under = try paths.join(settings.root, "build")
defer built_under.release()
let under = try paths.join(built_under.value, "luc-test")
defer under.release()
let output = try paths.join(under.value, program.name)
defer output.release()
let home = try paths.join(output.value, "home")
defer home.release()
let binary = try paths.join(output.value, program.name)
defer binary.release()
# a build beside others gets its share of the processors; a serial program's run, alone,
# gets them all
let run_share = 0 if program.serial else share
if build:
let scratch_root = try paths.join(built_under.value, "tests")
defer scratch_root.release()
let scratch = try paths.join(scratch_root.value, program.name)
defer scratch.release()
if try is_directory(home.value): try files.remove_tree((c.str)home.value)
for directory in [output.value, home.value, scratch.value]:
let problem = try make_directory(directory)
if problem.length > 0:
program.report = problem
return false
var words: str[]
defer free_words(words)
for word in ["build", program.source, "--native", "-o", binary.value]:
words = try append(words, [word])
if settings.diagnostic:
words = try append(words, ["--profile"])
words = try append(words, ["diagnostic"])
let compiler = settings.luce if program.luce else settings.base
let built = try execute(compiler, words, settings, none, share)
defer built.release()
if built.value.exit_code != 0:
program.report = try tail("build failed", built.value.output, built.value.error_output)
return false
if not run: return true
var nothing: str[]
let ran = try execute(binary.value, nothing, settings, Scratch(home = home.value, directory = program.directory), run_share)
defer ran.release()
_ = files.remove_tree((c.str)home.value) catch failure:
_ = failure
recover ()
let completed = ran.value
if completed.exit_code != 0:
var heading: u8[64]
program.report = try tail(try strings.format(heading, f"exit status {completed.exit_code}"), completed.output, completed.error_output)
return false
if let reason = skip_reason(completed.output):
program.skipped = true
program.report = try strings.copy(reason)
return true
let expected = try paths.join(program.directory, "expected")
defer expected.release()
if files.exists((c.str)expected.value):
let wanted = try files.read((c.str)expected.value)
defer wanted.release()
if (str)wanted.value != completed.output:
program.report = try tail("its standard output differs from its expected file", completed.output, completed.error_output)
return false
return true
## Make `path` a directory, replacing whatever stands there that is not one (a binary an
## older layout built, say); answer an empty text, or an owned report naming the path when
## it cannot be made.
func make_directory(path: str) -> str!:
if try is_directory(path): return ""
let terminated = try strings.copy(path)
defer strings.release(terminated)
if files.exists((c.str)terminated):
_ = files.remove_file((c.str)terminated) catch failure:
return try strings.join(["cannot make the directory ", path, ": ", failure.message], "")
_ = files.ensure_directory(path) catch failure:
return try strings.join(["cannot make the directory ", path, ": ", failure.message], "")
return ""
## What a test program runs with beyond the toolchain: its scratch home and its own
## directory.
struct Scratch:
var home: str
var directory: str
## The reason of the first line of `output` that starts `skip:`, if one does.
func skip_reason(output: str) -> str?:
var start: usize = 0
while start < output.length:
let end = strings.find_from(output, "\n", start) else output.length
let line = (str)output.bytes[start..<end]
if strings.starts_with(line, "skip:"): return strings.trim((str)line.bytes[5..])
start = end + 1
return none
## Run `program` with `words` from the package root (luc's own directory) and capture what
## it wrote, with LUCE and LUCE_BASE naming the compilers under test, so a program that
## builds or runs Luce uses the same toolchain. A test program also gets `scratch`: HOME,
## USERPROFILE and LUC_HOME are its home, and LUC_TEST_DIR its own directory. A `share`
## above zero is its LUCE_BASE_JOBS, beside other programs.
func execute(program: str, words: str[], settings: Settings, scratch: Scratch?, share: usize) -> interop.Owned[process.Completed]!:
var settings_texts: str[]
defer free_words(settings_texts)
let luce = try located(settings.luce)
defer strings.release(luce)
let base = try located(settings.base)
defer strings.release(base)
settings_texts = try append(settings_texts, ["LUCE=", luce])
settings_texts = try append(settings_texts, ["LUCE_BASE=", base])
if share > 0:
var share_text: u8[24]
settings_texts = try append(settings_texts, ["LUCE_BASE_JOBS=", try strings.format(share_text, f"{share}")])
if let given = scratch:
settings_texts = try append(settings_texts, ["HOME=", given.home])
settings_texts = try append(settings_texts, ["LUC_HOME=", given.home])
settings_texts = try append(settings_texts, ["LUC_TEST_DIR=", given.directory])
if os.windows: settings_texts = try append(settings_texts, ["USERPROFILE=", given.home])
var named: str[]
defer free_words(named)
named = try append(named, [program])
named = try append(named, [settings.root])
let arguments = try new c.str[words.length] --- in memory.heap
defer free(arguments) in memory.heap
for i in 0..<words.length: arguments[i] = (c.str)words[i]
let environment = try new c.str[settings_texts.length] --- in memory.heap
defer free(environment) in memory.heap
for i in 0..<settings_texts.length: environment[i] = (c.str)settings_texts[i]
return try process.run((c.str)named[0], arguments, (c.str)named[1], environment)
## Where `program` is: itself when it names a path, else the first directory of PATH that
## holds it, so a test program is told the very compiler luc uses whatever its directory.
## The caller releases the answer.
func located(program: str) -> str!:
if strings.contains(program, "/") or strings.contains(program, "\\"): return try strings.copy(program)
let search = os.env("PATH") else return try strings.copy(program)
let separator = ";" if os.windows else ":"
var start: usize = 0
while start <= search.length:
let end = strings.find_from(search, separator, start) else search.length
let directory = (str)search.bytes[start..<end]
start = end + 1
if directory.length == 0: continue
let candidate = try paths.join(directory, program)
defer candidate.release()
if files.exists((c.str)candidate.value): return try strings.copy(candidate.value)
if os.windows:
let windows = try strings.join([candidate.value, ".exe"], "")
if files.exists((c.str)windows): return windows
strings.release(windows)
return try strings.copy(program)
## `words` with one more, `pieces` joined into a NUL-terminated text it owns.
func append(words: str[], pieces: const str[]) -> str[]!:
let word = try strings.join(pieces, "")
errdefer strings.release(word)
let grown = try new str[words.length + 1] --- in memory.heap
for i in 0..<words.length: grown[i] = words[i]
grown[words.length] = word
if words.length > 0: free(words) in memory.heap
return grown
func free_words(words: str[]):
for word in words: strings.release(word)
if words.length > 0: free(words) in memory.heap
## A failed program's report: `heading`, then the last lines of what it wrote, its
## standard output and then its standard error.
func tail(heading: str, output: str, error_output: str) -> str!:
let combined = try strings.join([output, error_output], "")
defer strings.release(combined)
var start = combined.length
var lines: usize = 0
while start > 0:
if combined.bytes[start - 1] == '\n' and start < combined.length:
lines += 1
if lines >= tail_lines: break
start -= 1
let shown = (str)combined.bytes[start..]
let omitted = "...\n" if start > 0 else ""
return try strings.join([heading, "\n", omitted, shown], "")
## `report` indented under its FAIL line, each line once, ending with a newline.
func write_indented(report: str) -> !:
var start: usize = 0
while start < report.length:
let end = strings.find_from(report, "\n", start) else report.length
try io.stdout().write(f" {(str)report.bytes[start..<end]}\n")
start = end + 1
test "a short report shows all of the output":
let report = try tail("exit status 1", "a\nb\n", "c\n")
defer strings.release(report)
assert(report == "exit status 1\na\nb\nc\n")
test "a long report shows the last lines":
var lines: str[]
defer free_words(lines)
for i in 0..<30:
var line: u8[16]
lines = try append(lines, [try strings.format(line, f"{i}\n")])
let output = try strings.join(lines, "")
defer strings.release(output)
let report = try tail("exit status 2", output, "")
defer strings.release(report)
assert(strings.starts_with(report, "exit status 2\n...\n10\n11\n"))
assert(strings.ends_with(report, "\n29\n"))
test "only a dependency without a path is fetched":
# under build/, which every system has, rather than /tmp, which Windows has not
try files.ensure_directory("build")
let directory = try files.create_temporary_directory("build")
defer _ = files.remove_tree((c.str)directory.value) catch: recover ()
defer directory.release()
assert(not try registry_dependencies(directory.value))
let manifest = try paths.join(directory.value, "package.prisma")
defer manifest.release()
try files.write((c.str)manifest.value, b"#prisma 4.0\ndef package \"t\" {\n def dependency \"a\" {\n str path = \"../a\"\n }\n}\n")
assert(not try registry_dependencies(directory.value))
try files.write((c.str)manifest.value, b"#prisma 4.0\ndef package \"t\" {\n def dependency \"a\" {\n str path = \"../a\"\n }\n def dependency \"b\" {\n str owner = \"acme\"\n }\n}\n")
assert(try registry_dependencies(directory.value))
test "a program named by its path is not looked for":
let found = try located("/opt/luce/bin/luce-base")
defer strings.release(found)
assert(found == "/opt/luce/bin/luce-base")
test "a line starting skip: skips, with its reason":
let reason = skip_reason("checking\nskip: no Vulkan device\n") else error(0, "no reason")
assert(reason == "no Vulkan device")
assert(skip_reason("ok\nnot skip: here\n") == none)