dymokomi/luce-js
Source of release 0.6.0
#!/usr/bin/env python3
"""Build the ljs runner and run QuickJS's own tests with it.
The tests are upstream's tests/*.js (copied into tests/js/, MIT, see tests/js/LICENSE)
and examples/*.js (in examples/), run the way upstream's Makefile `test:` target runs them with qjs: one process per file,
from the repository root, `--std` for test_builtin.js. A file passes when ljs exits with
status 0 and, for the files in EXPECTED_OUTPUT, prints exactly that.
Files that cannot pass yet are listed in KNOWN_FAILURES with the reason; they are
reported as XFAIL and do not fail the run. A known failure that passes is reported as
XPASS so that its entry can be removed. Any other failure fails the run.
Usage: tests/run.py [--no-build] [--timeout SECONDS] [test_name ...]
"""
import argparse
import os
import subprocess
import sys
import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
WINDOWS = sys.platform == "win32"
LJS = os.path.join(ROOT, "build", "ljs.exe" if WINDOWS else "ljs")
# (file, extra ljs options), in the order of upstream's Makefile.
TESTS = [
("test_closure.js", []),
("test_language.js", []),
("test_builtin.js", ["--std"]),
("test_loop.js", []),
("test_bigint.js", []),
("test_cyclic_import.js", []),
("test_worker.js", []),
# not on Windows, as in upstream's Makefile: its `os` module has no exec, pipe or kill
("test_std.js", []),
("test_rw_handler.js", []),
# the native modules qjs loads as shared libraries are linked into ljs
("test_bjson.js", []),
("examples/test_point.js", []),
# not upstream: regressions found by test262 (tests/test262.py)
("test_conformance.js", []),
# not upstream: Atomics.wait/notify between a worker and the main thread
("test_worker_atomics.js", []),
# the other examples (upstream compiles them with qjsc), checked against their output
("examples/hello.js", []),
("examples/hello_module.js", []),
("examples/test_fib.js", []),
("examples/pi_bigint.js", []),
# not upstream: both sides of the interpreter's fast paths
("test_fast_paths.js", []),
]
# The files upstream's Makefile does not run on Windows (CONFIG_WIN32).
POSIX_ONLY = {"test_std.js", "test_rw_handler.js"}
# file -> the script arguments after the file name.
SCRIPT_ARGS = {
"examples/pi_bigint.js": ["100"],
}
# file -> what it must print on standard output.
EXPECTED_OUTPUT = {
"examples/hello.js": "Hello World\n",
"examples/hello_module.js": "Hello World\nfib(10)= 55\nmsg= { x: 1, tab: [ 1, 2, 3 ] }\n",
"examples/test_fib.js": "Hello World\nfib(10)= 55\n",
"examples/pi_bigint.js": "3.14159265358979323846264338327950288419716939937510"
"58209749445923078164062862089986280348253421170679\n",
}
# file -> why it cannot pass. Keep the classification: (a) a port bug, (b) a compiler bug,
# (c) a host feature deliberately not ported.
KNOWN_FAILURES = {
}
def build():
print("building build/ljs ...", flush=True)
os.makedirs(os.path.join(ROOT, "build"), exist_ok=True)
result = subprocess.run(["luce-base", "build", "tests/ljs.lucb", "-o", LJS], cwd=ROOT)
if result.returncode != 0:
print("FAIL: cannot build ljs")
sys.exit(1)
def first_error_line(output):
"""The first line that says what went wrong: a trap, an exception or an assertion."""
lines = [line for line in output.splitlines() if line.strip()]
for line in lines:
if line.startswith("trap:") or "Error" in line or "assertion" in line or "exception" in line:
return line.strip()
return lines[0].strip() if lines else ""
def run_test(name, options, timeout):
# a name with a directory is relative to the repository root (the examples)
# with '/' separators, which module names are resolved with, on Windows too
path = name if "/" in name else "tests/js/" + name
command = [LJS] + options + [path] + SCRIPT_ARGS.get(name, [])
started = time.time()
try:
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
errors="replace", timeout=timeout)
except subprocess.TimeoutExpired:
return False, f"timeout after {timeout}s", time.time() - started
output = result.stdout + result.stderr
expected = EXPECTED_OUTPUT.get(name)
if result.returncode == 0 and expected is not None and result.stdout != expected:
return False, f"unexpected output: {result.stdout[:60]!r}", time.time() - started
if result.returncode == 0:
return True, "", time.time() - started
reason = first_error_line(output)
if result.returncode < 0:
reason = f"killed by signal {-result.returncode}: {reason}"
return False, reason, time.time() - started
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--no-build", action="store_true", help="use the existing build/ljs")
parser.add_argument("--timeout", type=int, default=600, help="seconds per test file")
parser.add_argument("tests", nargs="*", help="run only these test files")
args = parser.parse_args()
if not args.no_build:
build()
selected = [t for t in TESTS if not args.tests or t[0] in args.tests
or t[0].removesuffix(".js") in args.tests]
if WINDOWS:
selected = [t for t in selected if t[0] not in POSIX_ONLY]
counts = {"PASS": 0, "FAIL": 0, "XFAIL": 0, "XPASS": 0}
for name, options in selected:
passed, reason, seconds = run_test(name, options, args.timeout)
known = KNOWN_FAILURES.get(name)
if passed:
status = "XPASS" if known else "PASS"
else:
status = "XFAIL" if known else "FAIL"
counts[status] += 1
line = f"{status:5} {name:28} {seconds:6.1f}s"
if reason:
line += f" {reason}"
print(line, flush=True)
if status == "XFAIL":
print(f"{'':7}known: {known}")
elif status == "XPASS":
print(f"{'':7}passes now: remove it from KNOWN_FAILURES")
print(f"\n{len(selected)} files: {counts['PASS']} passed, {counts['FAIL']} failed, "
f"{counts['XFAIL']} known failures, {counts['XPASS']} unexpected passes")
sys.exit(1 if counts["FAIL"] else 0)
if __name__ == "__main__":
main()