dymokomi/luce-js
Source of release 0.6.1
#!/usr/bin/env python3
"""Fetch test262, build the luce-js test262 runner and run the suite.
The runner is tests/run_test262/, a port of QuickJS's run-test262.c. This script does what
upstream's Makefile targets test2-bootstrap and test2 do:
- clone tc39/test262 at the commit QuickJS 2026-06-04 pins and apply QuickJS's
tests/test262.patch (tests/test262.patch here), into a directory outside the
repository (default: $LUCE_JS_TEST262 or ../.test262 next to the worktree), and point
the tests/test262 symlink at it;
- build build/run-test262 with `luce-base build ... --release`;
- run it from tests/ with tests/test262.conf, whose known-errors file is
tests/test262_errors.txt (QuickJS's own list; luce-js aims at exactly that list).
`--suite es5` runs the old ES5 suite instead, as upstream's `make test2o` does: the
es5-tests branch of tc39/test262 (QuickJS's doc/quickjs.texi) cloned into DIR/test262o and
linked from tests/test262o, run with tests/test262o.conf in the conf's default mode; its
known-errors file is tests/test262o_errors.txt (empty upstream).
Two ways to run:
- split (the default for the whole suite): the sorted test list is cut into index ranges
run by parallel single-threaded runner processes (`-T 1 -r - START STOP`). A Luce
trap ends the whole process it happens in, so a runner that dies is restarted after
the test it was running, and that test is reported as CRASH with the trap message.
The error lines of all processes are merged; with -u the merged list is written to
tests/test262_errors.txt in the runner's order.
- direct (`--direct`, or any runner option after `--`): one runner process with its
own threads, exactly `make test2`. Everything after `--` goes to the runner.
Usage:
tests/test262.py [--fetch-only] [--no-build] [--dir DIR] [-j JOBS] [-u] [-a|-s]
tests/test262.py [--no-build] --direct [-- runner options]
tests/test262.py --suite es5 [--direct] [-u]
Examples:
tests/test262.py # the whole suite, strict and sloppy, split
tests/test262.py -u # ... and update tests/test262_errors.txt
tests/test262.py -- -c test262.conf -a -E # only the tests of the errors file
tests/test262.py -- -c test262.conf -f test262/test/built-ins/Array/length.js
tests/test262.py -- -c test262.conf -a -d test262/test/built-ins/Proxy
tests/test262.py --suite es5 # the ES5 suite (make test2o)
"""
import argparse
import concurrent.futures
import os
import re
import subprocess
import sys
import tempfile
import threading
import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TESTS = os.path.join(ROOT, "tests")
RUNNER = os.path.join(ROOT, "build", "run-test262.exe" if sys.platform == "win32" else "run-test262")
LINK = os.path.join(TESTS, "test262")
# QuickJS 2026-06-04 Makefile: TEST262_COMMIT and TEST262_SINCE
TEST262_COMMIT = "5c8206929d81b2d3d727ca6aac56c18358c8d790"
TEST262_SINCE = "2025-09-01"
TEST262_URL = "https://github.com/tc39/test262.git"
# the old ES5 suite (QuickJS doc/quickjs.texi: git clone --single-branch --branch es5-tests)
TEST262O_BRANCH = "es5-tests"
class Suite:
"""What differs between the ES2015+ suite (test262) and the old ES5 one (test262o)."""
def __init__(self, name, conf, errors, marker, default_mode):
self.name = name # the checkout and link name
self.conf = conf # the runner configuration, in tests/
self.errors = errors # the known-errors file, in tests/
self.marker = marker # a file that exists in a complete checkout
self.default_mode = default_mode # the mode option of a split run
self.link = os.path.join(TESTS, name)
self.error_line = re.compile(r"^" + name + r"/\S+\.js:\d+: ")
SUITES = {
"test262": Suite("test262", "test262.conf", "test262_errors.txt", "features.txt", "-a"),
"es5": Suite("test262o", "test262o.conf", "test262o_errors.txt", "test/harness/sta.js", ""),
}
def default_dir():
return os.environ.get("LUCE_JS_TEST262") or os.path.join(os.path.dirname(ROOT), ".test262")
def run(command, cwd=None):
print("+ " + " ".join(command), flush=True)
result = subprocess.run(command, cwd=cwd)
if result.returncode != 0:
sys.exit(f"FAIL: {' '.join(command)} exited with {result.returncode}")
def fetch_es5(directory, suite):
"""Clone the es5-tests branch of test262, as QuickJS's documentation says."""
checkout = os.path.join(directory, suite.name)
if not os.path.exists(os.path.join(checkout, suite.marker)):
os.makedirs(directory, exist_ok=True)
run(["git", "clone", "--single-branch", "--branch", TEST262O_BRANCH, TEST262_URL, checkout])
if os.path.islink(suite.link) or os.path.exists(suite.link):
os.remove(suite.link)
os.symlink(checkout, suite.link)
def fetch(directory):
"""Clone and patch test262 as `make test2-bootstrap` does."""
patch = os.path.join(TESTS, "test262.patch")
checkout = os.path.join(directory, "test262")
if not os.path.exists(os.path.join(checkout, "features.txt")):
os.makedirs(directory, exist_ok=True)
# the files as committed, also on Windows (no CRLF conversion)
run(["git", "clone", "-c", "core.autocrlf=false", "--single-branch",
f"--shallow-since={TEST262_SINCE}", TEST262_URL, checkout])
run(["git", "checkout", "-q", TEST262_COMMIT], cwd=checkout)
else:
run(["git", "reset", "-q", "--hard", TEST262_COMMIT], cwd=checkout)
if sys.platform == "win32":
# Windows has no `patch`, and a directory symlink needs a privilege: git applies
# the patch and the link is a junction
run(["git", "apply", "-p1", patch], cwd=checkout)
if os.path.lexists(LINK):
os.rmdir(LINK)
run(["cmd", "/c", "mklink", "/J", LINK, checkout])
return
with open(patch, "rb") as f:
print(f"+ patch -p1 < {patch}", flush=True)
if subprocess.run(["patch", "-p1"], stdin=f, cwd=checkout).returncode != 0:
sys.exit("FAIL: cannot apply tests/test262.patch")
if os.path.islink(LINK) or os.path.exists(LINK):
os.remove(LINK)
os.symlink(checkout, LINK)
def build():
os.makedirs(os.path.dirname(RUNNER), exist_ok=True)
run(["luce-base", "build", "tests/run_test262", "-o", RUNNER, "--release"], cwd=ROOT)
def namelist_key(name):
"""The sort order of run-test262's namelist_cmp: digit runs compare as numbers."""
return [(0, int(part), "") if part.isdigit() else (1, 0, part)
for part in re.split(r"(\d+)", name) if part != ""]
def count_tests(suite):
"""An upper bound of the runner's test list: every .js file but the fixtures."""
count = 0
for _, _, files in os.walk(os.path.join(suite.link, "test")):
count += sum(1 for f in files if f.endswith(".js") and not f.endswith("_FIXTURE.js"))
return count
HEADER = re.compile(r"^(\d+): (\S+\.js)")
RESULT = re.compile(r"Result: (\d+)/(\d+) errors?(.*)")
class Totals:
def __init__(self):
self.lock = threading.Lock()
self.failed = self.count = self.new = self.changed = self.fixed = 0
self.lines = [] # error/status lines of the runners
self.crashes = [] # (index, file, reason)
def run_range(suite, mode_args, start, stop, totals, empty_errors, timeout):
"""Run the tests [start, stop]; restart after a test that kills the runner."""
while start <= stop:
command = [RUNNER, "-c", suite.conf] + mode_args + ["-T", "1", "-r", "-"]
if empty_errors:
# every error is reported as new, as with -u
command += ["-e", empty_errors]
command += [str(start), str(stop)]
timed_out = False
try:
result = subprocess.run(command, cwd=TESTS, capture_output=True, text=True,
errors="replace", timeout=timeout)
stdout, stderr, code = result.stdout, result.stderr, result.returncode
except subprocess.TimeoutExpired as e:
stdout = e.stdout.decode(errors="replace") if isinstance(e.stdout, bytes) else (e.stdout or "")
stderr = e.stderr.decode(errors="replace") if isinstance(e.stderr, bytes) else (e.stderr or "")
code, timed_out = None, True
last_index, last_file = None, None
lines = []
failed_runs = 0
for line in stdout.splitlines():
m = HEADER.match(line)
if m:
last_index, last_file = int(m.group(1)), m.group(2)
elif suite.error_line.match(line):
lines.append(line)
elif line == " FAILED":
failed_runs += 1
result_line = None if timed_out else RESULT.search(stderr)
with totals.lock:
totals.lines += lines
if result_line:
totals.failed += int(result_line.group(1))
totals.count += int(result_line.group(2))
rest = result_line.group(3)
for key in ("new", "changed", "fixed"):
m = re.search(r"(\d+) " + key, rest)
if m:
setattr(totals, key, getattr(totals, key) + int(m.group(1)))
else:
# the counts of a runner that died are lost; keep its failures
totals.failed += failed_runs
if result_line:
return
# the runner died (a trap or a signal) or hung in the test it announced last
if last_index is None:
sys.exit(f"FAIL: the runner died before its first test: {command}\n{stderr[-2000:]}")
if timed_out:
reason = f"TIMEOUT after {timeout}s"
else:
trap = [l for l in stderr.splitlines() if "trap:" in l]
others = [l for l in stderr.splitlines() if l.strip() and not re.fullmatch(r"[.!\-]*( \d+/\d+/\d+)?", l.strip())]
reason = trap[-1][trap[-1].index("trap:"):] if trap else (others[-1] if others else "")
reason += f" (exit status {code})"
with totals.lock:
totals.crashes.append((last_index, last_file, reason))
start = last_index + 1
UNEXPECTED = re.compile(r"^(\S+:\d+: (?:strict mode: )?)unexpected error: ")
def run_split(suite, mode_args, jobs, chunk, update, timeout):
total = count_tests(suite)
totals = Totals()
empty_errors = None
if update:
handle, empty_errors = tempfile.mkstemp(prefix="test262-errors-")
os.close(handle)
ranges = [(s, min(s + chunk - 1, total)) for s in range(0, total + 1, chunk)]
started = time.time()
with concurrent.futures.ThreadPoolExecutor(jobs) as pool:
futures = [pool.submit(run_range, suite, mode_args, s, e, totals, empty_errors, timeout) for s, e in ranges]
for n, future in enumerate(concurrent.futures.as_completed(futures), 1):
future.result()
print(f"\r{n}/{len(ranges)} ranges, {totals.failed}/{totals.count} errors, "
f"{len(totals.crashes)} crashes", end="", file=sys.stderr, flush=True)
print(file=sys.stderr)
crash_lines = [f"{name}:1: CRASH: {reason}" for _, name, reason in sorted(totals.crashes)]
if update:
os.remove(empty_errors)
# the lines the runner writes with -u: no "unexpected error: " prefix
errors = [UNEXPECTED.sub(r"\1", l) for l in totals.lines
if " previous error: " not in l and ": unknown feature: " not in l]
with open(os.path.join(TESTS, suite.errors), "w") as f:
for line in sorted(errors + crash_lines, key=namelist_key):
f.write(line + "\n")
else:
for line in sorted(totals.lines, key=namelist_key) + crash_lines:
print(line)
summary = f"Result: {totals.failed}/{totals.count} errors"
if totals.crashes:
summary += f", {len(totals.crashes)} crashes"
if not update:
for key in ("new", "changed", "fixed"):
if getattr(totals, key):
summary += f", {getattr(totals, key)} {key}"
print(summary + f" ({time.time() - started:.0f}s, {jobs} processes)", file=sys.stderr)
if update:
return 0
return 1 if totals.new or totals.changed or totals.fixed or totals.crashes else 0
def main():
argv = sys.argv[1:]
runner_args = None
if "--" in argv:
split = argv.index("--")
runner_args = argv[split + 1:]
argv = argv[:split]
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--suite", choices=["test262", "es5"], default="test262",
help="test262 (default) or es5, the old ES5 suite of make test2o")
parser.add_argument("--fetch-only", action="store_true", help="only clone and patch test262")
parser.add_argument("--no-build", action="store_true", help="use the existing build/run-test262")
parser.add_argument("--dir", default=default_dir(), help="where test262 is cloned")
parser.add_argument("--direct", action="store_true", help="one runner process (make test2)")
parser.add_argument("-j", "--jobs", type=int, default=max(1, (os.cpu_count() or 2) - 1))
parser.add_argument("--chunk", type=int, default=400, help="tests per runner process")
parser.add_argument("--timeout", type=int, default=600, help="seconds per runner process")
parser.add_argument("-u", "--update", action="store_true",
help="update the known-errors file (tests/test262_errors.txt)")
parser.add_argument("-a", dest="mode", action="store_const", const="-a", default=None,
help="strict and sloppy (the default of test262)")
parser.add_argument("-s", dest="mode", action="store_const", const="-s", help="strict only")
parser.add_argument("--default-mode", dest="mode", action="store_const", const="",
help="the mode of the configuration file (the default of es5)")
args = parser.parse_args(argv)
suite = SUITES[args.suite]
if not os.path.exists(os.path.join(suite.link, suite.marker)) or args.fetch_only:
if args.suite == "es5":
fetch_es5(args.dir, suite)
else:
fetch(args.dir)
if args.fetch_only:
return
if not args.no_build:
build()
if args.direct or runner_args is not None:
if runner_args is None:
# make test2 / make test2o
runner_args = ["-t", "-c", "test262.conf", "-a"] if args.suite == "test262" else \
["-t", "-m", "-c", suite.conf]
command = [RUNNER] + runner_args
print("+ (cd tests && " + " ".join(command) + ")", flush=True)
sys.exit(subprocess.run(command, cwd=TESTS).returncode)
mode = args.mode if args.mode is not None else suite.default_mode
mode_args = [mode] if mode else []
sys.exit(run_split(suite, mode_args, args.jobs, args.chunk, args.update, args.timeout))
if __name__ == "__main__":
main()