Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-svg

Source of release 0.9.1

0.9.1 / tools / suite.py

#!/usr/bin/env python3
"""Render resvg's test suite with luce-svg and compare every test with its reference PNG.

The suite is resvg's crates/resvg/tests/tests, the tests its integration/render.rs runs
(clone https://github.com/linebender/resvg into ../.donors/resvg; it is never committed
here). Each SVG is drawn 300 pixels wide by
build/suite (tools/suite.lucb), as resvg draws its references. A pixel differs when any
of its premultiplied RGBA8 channels is off by more than --threshold (40); a test passes
when at most --allowed (0.5 %) of its pixels differ. The fuzzy rule absorbs
anti-aliasing differences — tiny-skia samples four sub-scanlines a pixel, so its edge
coverage is quantised to quarters, up to an eighth (32 levels) from the exact area this
renderer computes — but not a missing or misplaced feature. --drawing measures the
Drawing renderer instead (the baseline). Text is set in the suite's fonts
(crates/resvg/tests/fonts, with resvg's generic families); --system-fonts sets it in the
system's fonts instead. Prints a pass table per feature directory and per category.

  build/env/bin/python tools/suite.py [--only painting/fill] [--save run.json]
                                      [--compare run.json] [--list-failing] [--jobs 8]
"""
import argparse, json, os, re, struct, subprocess, sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import numpy as np
from PIL import Image

ROOT = Path(__file__).resolve().parents[1]
SUITE = ROOT.parent / ".donors/resvg/crates/resvg/tests/tests"
FONTS = SUITE.parent / "fonts"
OUT = ROOT / "build/suite-out"
TOOL = ROOT / "build/suite"


def build():
    base = os.environ.get("LUCE_BASE", "luce-base")
    subprocess.run([base, "build", str(ROOT / "tools/suite.lucb"), "-o", str(TOOL), "--release"],
                   cwd=ROOT, check=True)


EXTRA = []


def render_chunk(job, paths, timeout):
    """Render `paths` into OUT/<job>-<n>.rgba, restarting past any file that crashes
    or hangs the tool. Returns the indices that crashed."""
    crashed = []
    start = 0
    while start < len(paths):
        listing = OUT / f"list-{job}.txt"
        listing.write_text("\n".join(str(p) for p in paths[start:]) + "\n")
        sub = OUT / f"job-{job}-{start}"
        sub.mkdir(exist_ok=True)
        try:
            run = subprocess.run([str(TOOL), str(listing), str(sub), *EXTRA], capture_output=True, timeout=timeout)
            ok = run.returncode == 0
        except subprocess.TimeoutExpired:
            ok = False
        done = start
        for n in range(len(paths) - start):
            target = sub / f"{n}.rgba"
            if target.exists():
                target.rename(OUT / f"{job}-{start + n}.rgba")
                done = start + n + 1
        if ok:
            break
        # The first file without output after the last one written is the culprit.
        culprit = done
        while culprit < len(paths) and (OUT / f"{job}-{culprit}.rgba").exists():
            culprit += 1
        if culprit >= len(paths):
            break
        crashed.append(culprit)
        start = culprit + 1
    return crashed


def load_ours(path):
    data = path.read_bytes()
    w, h = struct.unpack("<II", data[:8])
    return np.frombuffer(data[8:], dtype=np.uint8).reshape(h, w, 4)


def premultiplied(rgba):
    a = rgba[:, :, 3:4].astype(np.float32) / 255.0
    out = rgba.astype(np.float32)
    out[:, :, :3] *= a
    return out


def compare(ours, reference, threshold, allowed):
    if ours is None:
        return False, 1.0
    if ours.shape != reference.shape:
        return False, 1.0
    diff = np.abs(premultiplied(ours) - premultiplied(reference)).max(axis=2)
    bad = float((diff > threshold).mean())
    return bad <= allowed, bad


def main():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--only", default="", help="run tests whose path contains this")
    p.add_argument("--threshold", type=int, default=40)
    p.add_argument("--allowed", type=float, default=0.005)
    p.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
    p.add_argument("--timeout", type=int, default=120)
    p.add_argument("--save", default="")
    p.add_argument("--compare", default="")
    p.add_argument("--list-failing", action="store_true")
    p.add_argument("--no-build", action="store_true")
    p.add_argument("--drawing", action="store_true", help="draw with the Drawing renderer (the baseline)")
    p.add_argument("--system-fonts", action="store_true", help="set text in the system's fonts, not the suite's")
    p.add_argument("--tool", default="", help="a renderer built elsewhere taking the same arguments")
    p.add_argument("--tool-arg", action="append", default=[], help="an argument the tool takes after them")
    a = p.parse_args()
    if not SUITE.exists():
        sys.exit(f"no suite at {SUITE}: clone https://github.com/linebender/resvg into ../.donors/resvg")
    global TOOL
    if a.tool:
        TOOL = Path(a.tool).resolve()
    elif not a.no_build:
        build()
    EXTRA.extend(a.tool_arg)
    if a.drawing:
        EXTRA.append("--drawing")
    elif not a.system_fonts and not a.tool:
        EXTRA.extend(["--fonts", str(FONTS)])
    OUT.mkdir(parents=True, exist_ok=True)
    for stale in OUT.rglob("*.rgba"):
        stale.unlink()
    # The suite is what resvg's integration tests run (a few files there are not run).
    listing = SUITE.parent / "integration/render.rs"
    listed = set(re.findall(r'render\("tests/([^"]*)"\)', listing.read_text())) if listing.exists() else None
    tests = sorted(t for t in SUITE.rglob("*.svg")
                   if a.only in str(t.relative_to(SUITE)) and (listed is None or str(t.relative_to(SUITE))[:-4] in listed))
    chunks = [tests[i::a.jobs] for i in range(a.jobs)]
    with ThreadPoolExecutor(a.jobs) as pool:
        crashes = list(pool.map(lambda j: render_chunk(j, chunks[j], a.timeout), range(len(chunks))))
    results = {}
    for job, chunk in enumerate(chunks):
        for n, test in enumerate(chunk):
            name = str(test.relative_to(SUITE))[:-4]
            path = OUT / f"{job}-{n}.rgba"
            ours = load_ours(path) if path.exists() else None
            reference = np.asarray(Image.open(test.with_suffix(".png")).convert("RGBA"))
            ok, bad = compare(ours, reference, a.threshold, a.allowed)
            results[name] = {"pass": ok, "bad": round(bad, 5), "crash": n in crashes[job]}
            if ours is not None and not ok:
                Image.fromarray(np.ascontiguousarray(ours)).save(OUT / (name.replace("/", "_") + ".png"))
    table = {}
    for name, r in results.items():
        feature = "/".join(name.split("/")[:2])
        row = table.setdefault(feature, [0, 0, 0])
        row[0] += r["pass"]
        row[1] += 1
        row[2] += r["crash"]
    categories = {}
    print(f"{'feature':44} {'pass':>5} {'of':>5}")
    for feature in sorted(table):
        passed, total, crashed = table[feature]
        print(f"{feature:44} {passed:5} {total:5}" + (f"  ({crashed} crashed)" if crashed else ""))
        cat = categories.setdefault(feature.split("/")[0], [0, 0])
        cat[0] += passed
        cat[1] += total
    print()
    for cat in sorted(categories):
        print(f"{cat:44} {categories[cat][0]:5} {categories[cat][1]:5}")
    passed = sum(r["pass"] for r in results.values())
    print(f"{'total':44} {passed:5} {len(results):5}")
    if a.compare:
        before = json.loads(Path(a.compare).read_text())
        gained = sorted(n for n, r in results.items() if r["pass"] and not before.get(n, {}).get("pass"))
        lost = sorted(n for n, r in results.items() if not r["pass"] and before.get(n, {}).get("pass"))
        print(f"\nnewly passing: {len(gained)}  newly failing: {len(lost)}")
        for n in lost:
            print("  LOST", n, results[n]["bad"])
    if a.list_failing:
        for n, r in sorted(results.items()):
            if not r["pass"]:
                print(f"  fail {r['bad']:.4f} {'CRASH ' if r['crash'] else ''}{n}")
    if a.save:
        Path(a.save).write_text(json.dumps(results, indent=0, sort_keys=True))


if __name__ == "__main__":
    main()