Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-js

Source of release 0.6.1

0.6.1 / tests / bench.py

#!/usr/bin/env python3
"""Time the benchmarks of tests/bench/ with ljs (built --release) against C QuickJS's qjs.

Each benchmark runs `--runs` times with each engine; the best wall time is reported, with
ljs/qjs as the ratio. `--save FILE` writes the times as JSON, `--compare FILE` adds a
column with the ljs times of an earlier run and the speed-up since.

`--micro` runs upstream's tests/bench/microbench.js (QuickJS's tests/microbench.js) with
each engine instead, and reports its time per operation of each microbenchmark (the names
given select microbenchmarks by prefix, as microbench.js does), with the geometric mean of
the ratios. A full run takes about four minutes.

QJS names the qjs binary and LUCE_BASE the compiler (defaults: qjs, luce-base on the PATH).

Usage: tests/bench.py [--no-build] [--qjs PATH] [--runs N] [--save F] [--compare F] [bench ...]
       tests/bench.py --micro [--no-build] [--qjs PATH] [--save F] [--compare F] [name ...]
"""

import argparse
import json
import math
import os
import subprocess
import sys
import time

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BENCH = os.path.join(ROOT, "tests", "bench")
LJS = os.path.join(ROOT, "build", "ljs-release")
QJS = os.environ.get("QJS", "qjs")
# the compiler that builds ljs (LUCE_BASE, default luce-base on the PATH)
LUCE_BASE = os.environ.get("LUCE_BASE", "luce-base")


def build():
    print("building build/ljs-release ...", flush=True)
    os.makedirs(os.path.join(ROOT, "build"), exist_ok=True)
    r = subprocess.run([LUCE_BASE, "build", "tests/ljs.lucb", "-o", LJS, "--release"], cwd=ROOT)
    if r.returncode != 0:
        sys.exit("FAIL: cannot build ljs")


def best_time(cmd, runs):
    best = None
    for _ in range(runs):
        t0 = time.perf_counter()
        r = subprocess.run(cmd, cwd=BENCH, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
        dt = time.perf_counter() - t0
        if r.returncode != 0:
            sys.exit(f"FAIL: {' '.join(cmd)}: {r.stderr.decode(errors='replace').strip()}")
        best = dt if best is None else min(best, dt)
    return best


def micro_times(engine, names):
    """The time per operation (ns) of each microbenchmark of microbench.js run by `engine`."""
    cmd = [engine, "--std", "microbench.js", "-s", ""] + names
    r = subprocess.run(cmd, cwd=BENCH, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if r.returncode != 0:
        sys.exit(f"FAIL: {' '.join(cmd)}: {r.stderr.decode(errors='replace').strip()}")
    times = {}
    for line in r.stdout.decode().splitlines()[1:]:
        cols = line.split()
        if len(cols) == 3 and cols[0] != "total":
            times[cols[0]] = float(cols[2])
    return times


def micro(args, before):
    q = micro_times(args.qjs, args.names)
    l = micro_times(LJS, args.names)
    head = f"{'microbenchmark':<24} {'qjs ns':>9} {'ljs ns':>9} {'ljs/qjs':>8}"
    if before:
        head += f" {'before ns':>10} {'speed-up':>9}"
    print(head)
    results = {}
    logs = []
    for name in q:
        if name not in l:
            continue
        results[name] = {"qjs": q[name], "ljs": l[name]}
        ratio = l[name] / q[name]
        logs.append(math.log(ratio))
        line = f"{name:<24} {q[name]:9.2f} {l[name]:9.2f} {ratio:8.1f}"
        if name in before:
            b = before[name]["ljs"]
            line += f" {b:10.2f} {b / l[name]:9.2f}"
        print(line)
    if logs:
        print(f"{'geometric mean':<24} {'':9} {'':9} {math.exp(sum(logs) / len(logs)):8.2f}")
    return results


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-build", action="store_true")
    ap.add_argument("--qjs", default=QJS)
    ap.add_argument("--runs", type=int, default=3)
    ap.add_argument("--save")
    ap.add_argument("--compare")
    ap.add_argument("--micro", action="store_true")
    ap.add_argument("names", nargs="*")
    args = ap.parse_args()
    if os.sep in args.qjs:
        # the benchmarks run in tests/bench
        args.qjs = os.path.abspath(args.qjs)
    if not args.no_build:
        build()
    before = {}
    if args.compare:
        with open(args.compare) as f:
            before = json.load(f)
    if args.micro:
        results = micro(args, before)
        if args.save:
            with open(args.save, "w") as f:
                json.dump(results, f, indent=1)
        return
    names = args.names or sorted(f[:-3] for f in os.listdir(BENCH)
                                 if f.endswith(".js") and f != "microbench.js")
    results = {}
    head = f"{'benchmark':<18} {'qjs s':>7} {'ljs s':>7} {'ljs/qjs':>8}"
    if before:
        head += f" {'before s':>9} {'speed-up':>9}"
    print(head)
    for name in names:
        q = best_time([args.qjs, name + ".js"], args.runs)
        l = best_time([LJS, name + ".js"], args.runs)
        results[name] = {"qjs": q, "ljs": l}
        line = f"{name:<18} {q:7.3f} {l:7.3f} {l / q:8.1f}"
        if name in before:
            b = before[name]["ljs"]
            line += f" {b:9.3f} {b / l:9.2f}"
        print(line, flush=True)
    if args.save:
        with open(args.save, "w") as f:
            json.dump(results, f, indent=1)


if __name__ == "__main__":
    main()