Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-xml

Source of release 0.3.0

0.3.0 / tools / xmlconf.py

#!/usr/bin/env python3
"""Run the W3C XML Conformance Test Suite against luce-xml and print pass rates.

The suite (xmlts20130923, https://www.w3.org/XML/Test/) is unpacked into
../.donors/xmlconf (never committed here). Every XML 1.0 and Namespaces 1.0 test of a
non-validating processor runs, Fifth Edition rules where an edition is named (XML 1.1
and Namespaces 1.1 tests are left out):

- valid and invalid documents must be accepted (invalid ones are well-formed: only a
  validating parser rejects them); where the suite gives the canonical output, ours
  must match it byte for byte;
- not-wf documents must be rejected;
- error documents may go either way, and are only counted.

Tests marked NAMESPACE="no" run with namespaces off. External entities are read from
the suite's files (the parser asks the tool, which reads them). A document in an
encoding luce-xml does not decode is transcoded here with Python's codecs and parsed
again as UTF-8, as a browser would hand it over.

--no-external leaves every external entity unread, as a browser does; then only the
not-wf documents whose error is in the document itself (ENTITIES="none") must be
rejected, and canonical outputs are not compared (the suite's assume a reading
processor).

  python3 tools/xmlconf.py [--suite DIR] [--list-failing] [--only xmltest] [--no-external]

LUCE_BASE names the compiler (default: luce-base on PATH).
"""
import argparse, os, re, subprocess, sys, tempfile
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
SUITE = ROOT.parent / ".donors/xmlconf"
TOOL = ROOT / "build/xmlconf"
COLLECTIONS = ["xmltest", "sun", "ibm", "oasis", "japanese", "eduni"]
INDEXES = [
    "xmltest/xmltest.xml", "japanese/japanese.xml", "sun/sun-valid.xml", "sun/sun-invalid.xml",
    "sun/sun-not-wf.xml", "sun/sun-error.xml", "oasis/oasis.xml", "ibm/ibm_oasis_invalid.xml",
    "ibm/ibm_oasis_not-wf.xml", "ibm/ibm_oasis_valid.xml", "eduni/errata-2e/errata2e.xml",
    "eduni/namespaces/1.0/rmt-ns10.xml", "eduni/errata-3e/errata3e.xml",
    "eduni/namespaces/errata-1e/errata1e.xml", "eduni/errata-4e/errata4e.xml", "eduni/misc/ht-bh.xml",
]


def build():
    base = os.environ.get("LUCE_BASE", "luce-base")
    TOOL.parent.mkdir(exist_ok=True)
    subprocess.run([base, "build", str(ROOT / "tools/xmlconf.lucb"), "-o", str(TOOL), "--release"],
                   cwd=ROOT, check=True)


def tests(suite):
    """Every test the run covers: (collection, attributes, path, output path or None)."""
    found = []
    for index in INDEXES:
        path = suite / index
        text = path.read_text(encoding="utf-8", errors="replace")
        for match in re.finditer(r"<TEST\b([^>]*)>", text):
            attributes = {key: double or single for key, double, single in
                          re.findall(r"""([A-Z]+)\s*=\s*(?:"([^"]*)"|'([^']*)')""", match.group(1))}
            if attributes.get("VERSION", "1.0") not in ("1.0", ""):
                continue
            if attributes.get("RECOMMENDATION", "XML1.0") in ("XML1.1", "NS1.1"):
                continue
            if "EDITION" in attributes and "5" not in attributes["EDITION"].split():
                continue
            document = path.parent / attributes["URI"]
            output = path.parent / attributes["OUTPUT"] if "OUTPUT" in attributes else None
            found.append((index.split("/")[0], attributes, document, output))
    return found


def run_batch(jobs, timeout=600):
    """Run the tool over `jobs` [(flags, path)]: per job, ("ok", output bytes) or
    (status, message). Restarts past a job that crashes the tool."""
    results = [None] * len(jobs)
    with tempfile.TemporaryDirectory() as directory:
        start = 0
        while start < len(jobs):
            listing = Path(directory) / "list.txt"
            listing.write_text("".join(f"{flags or '-'} {path}\n" for flags, path in jobs[start:]))
            out = Path(directory) / f"out-{start}"
            out.mkdir()
            try:
                run = subprocess.run([str(TOOL), str(listing), str(out)], capture_output=True, timeout=timeout)
                lines = run.stdout.decode("utf-8", "replace").splitlines()
                crashed = run.returncode != 0
                trailer = run.stderr.decode("utf-8", "replace").strip()
            except subprocess.TimeoutExpired as expired:
                lines = (expired.stdout or b"").decode("utf-8", "replace").splitlines()
                crashed, trailer = True, "timeout"
            done = start
            for line in lines:
                number, _, rest = line.partition(" ")
                if not number.isdigit():
                    continue
                index = start + int(number)
                status, _, message = rest.partition(" ")
                if status == "ok":
                    results[index] = ("ok", (out / f"{number}.xml").read_bytes())
                else:
                    results[index] = (status, message)
                done = index + 1
            if crashed and done < len(jobs):
                results[done] = ("crash", trailer[-300:])
                done += 1
            start = done
    return results


CODECS = {"shift_jis": "shift_jis", "euc-jp": "euc_jp", "iso-2022-jp": "iso2022_jp", "big5": "big5",
          "euc-kr": "euc_kr", "gb2312": "gb2312", "iso-8859-2": "iso8859_2", "windows-1252": "cp1252",
          "koi8-r": "koi8_r", "iso-10646-ucs-2": "utf_16", "ucs-2": "utf_16", "utf-16": "utf_16"}


def transcode(path, label):
    """The document at `path`, decoded as `label` and written as UTF-8 beside it (so its
    relative entities still resolve); None if Python has no codec for it."""
    codec = CODECS.get(label.lower(), label.lower())
    try:
        text = Path(path).read_bytes().decode(codec)
    except (LookupError, UnicodeDecodeError):
        return None
    target = Path(path).with_name(f".utf8-{Path(path).name}")
    target.write_bytes(text.encode("utf-8"))
    return target


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--suite", type=Path, default=SUITE)
    parser.add_argument("--list-failing", action="store_true")
    parser.add_argument("--only", default=None)
    parser.add_argument("--no-build", action="store_true")
    parser.add_argument("--no-external", action="store_true")
    options = parser.parse_args()
    if not options.no_build:
        build()
    cases = [case for case in tests(options.suite) if options.only in (None, case[0])]
    if options.no_external:
        cases = [(c, a, d, None) for c, a, d, _ in cases
                 if a["TYPE"] != "not-wf" or a.get("ENTITIES", "none") == "none"]
    unread = "x" if options.no_external else ""
    jobs = [(("n" if case[1].get("NAMESPACE") == "no" else "") + unread, str(case[2])) for case in cases]
    results = run_batch(jobs)
    # Documents in encodings luce-xml leaves to its caller: transcode and parse again.
    retry = [i for i, result in enumerate(results) if result and result[0] == "encoding"]
    made = []
    if retry:
        again = []
        for i in retry:
            target = transcode(cases[i][2], results[i][1])
            if target is None:
                continue
            made.append(target)
            again.append((i, (jobs[i][0] + "u", str(target))))
        for (i, _), result in zip(again, run_batch([job for _, job in again])):
            results[i] = result
    for target in made:
        target.unlink()
    table = defaultdict(lambda: defaultdict(lambda: [0, 0]))
    failing = []
    for (collection, attributes, document, output), result in zip(cases, results):
        kind = attributes["TYPE"]
        status = result[0] if result else "crash"
        name = f"{collection} {attributes['ID']} ({document.relative_to(options.suite)})"
        if status == "crash":
            failing.append(f"CRASH {name}: {result[1] if result else ''}")
        if kind in ("valid", "invalid"):
            row = table[collection][kind]
            row[1] += 1
            if status == "ok":
                row[0] += 1
                if output is not None:
                    compared = table[collection]["output"]
                    compared[1] += 1
                    if output.read_bytes() == result[1]:
                        compared[0] += 1
                    else:
                        failing.append(f"OUTPUT {name}\n    expected {output.read_bytes()[:300]!r}\n    actual   {result[1][:300]!r}")
            else:
                failing.append(f"REJECTED {kind} {name}: {result[1] if result else ''}")
        elif kind == "not-wf":
            row = table[collection]["not-wf"]
            row[1] += 1
            if status in ("error", "encoding") and not result[1].startswith("listener"):
                row[0] += 1
            else:
                failing.append(f"ACCEPTED not-wf {name}")
        else:
            row = table[collection]["error"]
            row[1] += 1
            if status == "error":
                row[0] += 1
    print(f"{'collection':<10} {'valid':>12} {'invalid':>12} {'not-wf':>12} {'output':>12} {'error*':>10}")
    totals = defaultdict(lambda: [0, 0])
    for collection in COLLECTIONS:
        if collection not in table:
            continue
        cells = []
        for kind in ("valid", "invalid", "not-wf", "output", "error"):
            passed, total = table[collection][kind]
            totals[kind][0] += passed
            totals[kind][1] += total
            cells.append(f"{passed}/{total}" if total else "-")
        print(f"{collection:<10} {cells[0]:>12} {cells[1]:>12} {cells[2]:>12} {cells[3]:>12} {cells[4]:>10}")
    cells = [f"{totals[k][0]}/{totals[k][1]}" for k in ("valid", "invalid", "not-wf", "output", "error")]
    print(f"{'all':<10} {cells[0]:>12} {cells[1]:>12} {cells[2]:>12} {cells[3]:>12} {cells[4]:>10}")
    print("valid, invalid: accepted; not-wf: rejected; output: canonical output matches;")
    print("error*: optional errors the parser reports (either answer conforms)")
    if options.list_failing:
        for line in failing:
            print(line)
    return 0 if not failing else 1


if __name__ == "__main__":
    sys.exit(main())