Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-crypto

Source of release 0.6.0

0.6.0 / tools / openssl_vectors.py

#!/usr/bin/env python3
"""Convert OpenSSL EVP test data into line files under tests/vectors/openssl/.

Usage: tools/openssl_vectors.py OPENSSL_CHECKOUT

Reads test/recipes/30-test_evp_data/*.txt of OpenSSL (Apache-2.0) at the
revision in REVISION. Only data is converted; no OpenSSL code is copied.
Every stanza of a family this package implements is written, including error
cases (RESULT names OpenSSL's expected error, "ok" otherwise); stanzas for
algorithms or modes this package does not implement are counted in
SUMMARY.txt so the README can list them. Fields are hex, "-" for empty and
"absent" for a parameter the stanza does not set.

  digest.txt   ALG OUTPUT CHUNK... (CHUNK is INPUT:NCOPY:COUNT, fed in order)
  hmac.txt     ALG KEY INPUT OUTPUT RESULT
  hkdf.txt     MODE ALG IKM SALT INFO OUTPUT RESULT
  cipher.txt   CIPHER OPERATION RESULT KEY IV AAD PLAIN CIPHERTEXT TAG
  x25519.txt   derive RESULT PRIVATE PEER SHARED | keypair RESULT PRIVATE PUBLIC
  ecdh.txt     derive RESULT PRIVATE PEERX PEERY SHARED | keypair RESULT PRIVATE X Y   (P-256)
  ecdsa.txt    verify CURVE HASH X Y DIGEST SIG RESULT | digestverify CURVE HASH X Y MSG SIG RESULT
               | digestsign CURVE HASH PRIVATE MSG SIG | keypair CURVE RESULT PRIVATE X Y
  rsa.txt      PADDING N E HASH MGFHASH SALT PREHASHED INPUT SIG RESULT
               (PADDING pkcs1 or pss; SALT a byte count or "auto"; PREHASHED 1 when
               INPUT is the digest, 0 when it is the message)
"""
import base64
import collections
import hashlib
from pathlib import Path
import sys

REVISION = "adb795d9b166b7342ad1227b6241f3d31d973438"
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "tests/vectors/openssl"
DIGESTS = {"sha1": "sha1", "sha224": "sha224", "sha256": "sha256", "sha384": "sha384", "sha512": "sha512",
           "sha2-224": "sha224", "sha2-256": "sha256", "sha2-384": "sha384", "sha2-512": "sha512"}


def stanzas(path):
    """Blank-line separated stanzas as ordered (key, value) lists. PEM blocks
    following a key line become that key's value."""
    lines, out, current, at = path.read_text().splitlines(), [], [], 0
    while at < len(lines):
        line = lines[at]
        if line.startswith("#"):
            at += 1
            continue
        if not line.strip():
            if current:
                out.append(current)
            current = []
            at += 1
            continue
        key, _, value = line.partition("=")
        key, value = key.strip(), value.strip()
        if at + 1 < len(lines) and lines[at + 1].startswith("-----BEGIN"):
            pem = []
            at += 1
            while not lines[at].startswith("-----END"):
                pem.append(lines[at])
                at += 1
            pem.append(lines[at])
            current.append((key, value, "\n".join(pem)))
        else:
            current.append((key, value))
        at += 1
    if current:
        out.append(current)
    return out


def get(stanza, key, default=None):
    for item in stanza:
        if item[0] == key:
            return item[1]
    return default


def value_hex(text):
    """A test value: a quoted string, or hex."""
    if text is None:
        return "absent"
    if text.startswith('"'):
        return text.strip('"').encode().hex() or "-"
    return text.lower() or "-"


def input_hex(stanza):
    """All Input values of a stanza, concatenated (OpenSSL feeds them in turn)."""
    parts = [value_hex(item[1]) for item in stanza if item[0] == "Input"]
    joined = "".join(p for p in parts if p != "-")
    return joined or ("-" if parts else "absent")


def ctrl_hex(text):
    """A Ctrl value such as hexkey:00ff, salt:text or digest:SHA256."""
    if text is None:
        return "absent"
    name, _, value = text.partition(":")
    if name.startswith("hex"):
        return value.lower() or "-"
    return value.encode().hex() or "-"


def provider_only(stanza):
    """Stanzas that only run in the FIPS provider test OpenSSL's FIPS policy."""
    available = get(stanza, "Availablein")
    return available is not None and "default" not in available


# ---- Keys ------------------------------------------------------------------

def tlv(data, at=0):
    """(tag, content, end) of the DER element at `at`."""
    tag, first = data[at], data[at + 1]
    at += 2
    if first & 0x80:
        count = first & 0x7F
        first = int.from_bytes(data[at:at + count], "big")
        at += count
    return tag, data[at:at + first], at + first


def children(data):
    out, at = [], 0
    while at < len(data):
        tag, content, end = tlv(data, at)
        out.append((tag, content, data[at:end]))
        at = end
    return out


OIDS = {"2b656e": "x25519", "2b656f": "x448", "2b6570": "ed25519", "2b6571": "ed448", "2a8648ce3d030107": "p256", "2b81040022": "p384",
        "2b81040023": "p521", "2a864886f70d010101": "rsa", "2a864886f70d01010a": "rsa-pss",
        "2a8648ce3d0201": "ec"}


def curve_name(oid):
    return OIDS.get(oid.hex(), "other-curve")


def pem_der(pem):
    body = "".join(line for line in pem.splitlines() if not line.startswith("-----"))
    return base64.b64decode(body)


class Key:
    """kind (x25519, p256, p384, rsa, ...), private scalar, public bytes or (n, e)."""

    def __init__(self, kind, private=None, public=None):
        self.kind, self.private, self.public = kind, private, public


def parse_key(pem, private):
    der = pem_der(pem)
    label = pem.splitlines()[0]
    parts = children(tlv(der)[1])
    if "RSA PRIVATE KEY" in label:
        fields = [c[1] for c in parts]
        return Key("rsa", None, (fields[1], fields[2]))
    if "EC PRIVATE KEY" in label:
        return ec_private(tlv(der)[1], None)
    if private:
        algorithm = children(parts[1][1])
        kind = curve_name(algorithm[0][1])
        inner = parts[2][1]
        if kind in ("x25519", "x448"):
            return Key(kind, tlv(inner)[1])
        if kind == "ec":
            named = curve_name(algorithm[1][1]) if algorithm[1][0] == 6 else "explicit-parameters"
            return ec_private(tlv(inner)[1], named)
        if kind in ("rsa", "rsa-pss"):
            fields = [c[1] for c in children(tlv(inner)[1])]
            return Key(kind, None, (fields[1], fields[2]))
        return Key(kind)
    algorithm = children(parts[0][1])
    kind = curve_name(algorithm[0][1])
    bits = parts[1][1][1:]
    if kind == "ec":
        return Key(curve_name(algorithm[1][1]) if algorithm[1][0] == 6 else "explicit-parameters", None, bits)
    if kind in ("rsa", "rsa-pss"):
        fields = [c[1] for c in children(tlv(bits)[1])]
        return Key(kind, None, (fields[0], fields[1]))
    return Key(kind, None, bits)


def ec_private(sequence, curve):
    fields = children(sequence)
    scalar, public = fields[1][1], None
    for tag, content, _ in fields[2:]:
        if tag == 0xA0:
            inner = tlv(content)
            curve = curve_name(inner[1]) if inner[0] == 6 else "explicit-parameters"
        elif tag == 0xA1:
            public = tlv(content)[1][1:]
    return Key(curve or "explicit-parameters", scalar, public)


def keys_of(stanzas_list):
    keys = {}
    for stanza in stanzas_list:
        for item in stanza:
            if item[0] in ("PrivateKey", "PublicKey") and len(item) == 3:
                try:
                    keys[item[1]] = parse_key(item[2], item[0] == "PrivateKey")
                except (IndexError, ValueError):
                    keys[item[1]] = Key("unparsable")
            elif item[0] in ("PrivateKeyRaw", "PublicKeyRaw"):
                name, kind, value = item[1].split(":", 2)
                key = Key(kind.lower())
                if item[0] == "PrivateKeyRaw":
                    key.private = bytes.fromhex(value)
                else:
                    key.public = bytes.fromhex(value)
                keys[name] = key
    return keys


CURVES = {
    "p256": (2**256 - 2**224 + 2**192 + 2**96 - 1,
             0x6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296,
             0x4FE342E2FE1A7F9B8EE7EB4A7C0F9E162BCE33576B315ECECBB6406837BF51F5, 32),
    "p384": (2**384 - 2**128 - 2**96 + 2**32 - 1,
             0xAA87CA22BE8B05378EB1C71EF320AD746E1D3B628BA79B9859F741E082542A385502F25DBF55296C3A545E3872760AB7,
             0x3617DE4A96262C6F5D9E98BF9292DC29F8F41DBD289A147CE9DA3113B5F0B8C00A60B1CE1D7E819D7A431D7C90EA0E5F, 48),
}


def ec_public(curve, scalar):
    """scalar * G in affine coordinates, as 04 || X || Y (public data, Python ints)."""
    p, gx, gy, width = CURVES[curve]
    def add(a, b):
        if a is None:
            return b
        if a[0] == b[0] and (a[1] + b[1]) % p == 0:
            return None
        if a == b:
            slope = (3 * a[0] * a[0] - 3) * pow(2 * a[1], -1, p) % p
        else:
            slope = (b[1] - a[1]) * pow(b[0] - a[0], -1, p) % p
        x = (slope * slope - a[0] - b[0]) % p
        return x, (slope * (a[0] - x) - a[1]) % p
    result, addend, k = None, (gx, gy), int.from_bytes(scalar, "big")
    while k:
        if k & 1:
            result = add(result, addend)
        addend = add(addend, addend)
        k >>= 1
    return b"\x04" + result[0].to_bytes(width, "big") + result[1].to_bytes(width, "big")


def point_xy(key):
    """The public point as X and Y hex for P-256/P-384, from the key or its
    private scalar; None when it is not an uncompressed point."""
    public = key.public if key.public else (ec_public(key.kind, key.private) if key.private else None)
    width = CURVES[key.kind][3]
    if public is None or len(public) != 1 + 2 * width or public[0] != 4:
        return None
    return public[1:1 + width].hex(), public[1 + width:].hex()


def scalar_hex(key):
    return key.private.rjust(CURVES[key.kind][3], b"\0").hex()


def ecdh(source, summary):
    lines = []
    blocks = stanzas(source / "evppkey_ecdh.txt")
    keys = keys_of(blocks)
    for stanza in blocks:
        if provider_only(stanza):
            if get(stanza, "Derive"):
                summary["evppkey_ecdh.txt: FIPS-provider-only derive checks"] += 1
            continue
        if get(stanza, "Derive"):
            mine = keys[get(stanza, "Derive")]
            peer_name = get(stanza, "PeerKey") or get(stanza, "PeerKeyValidate")
            peer = keys[peer_name]
            if mine.kind != "p256":
                summary[f"evppkey_ecdh.txt: {mine.kind} derive (only P-256 ECDH is implemented)"] += 1
                continue
            xy = point_xy(peer) if peer.kind == "p256" else None
            result = get(stanza, "Result", "ok")
            if xy is None:
                lines.append(f"derive {result} {scalar_hex(mine)} - - {value_hex(get(stanza, 'SharedSecret'))}")
            else:
                lines.append(f"derive {result} {scalar_hex(mine)} {xy[0]} {xy[1]} {value_hex(get(stanza, 'SharedSecret'))}")
        elif get(stanza, "PrivPubKeyPair"):
            mine, theirs = (keys[n] for n in get(stanza, "PrivPubKeyPair").split(":"))
            if mine.kind != "p256":
                summary[f"evppkey_ecdh.txt: {mine.kind} key pair (only P-256 ECDH is implemented)"] += 1
                continue
            xy = point_xy(theirs)
            lines.append(f"keypair {get(stanza, 'Result', 'ok')} {scalar_hex(mine)} {xy[0]} {xy[1]}")
    return lines


def ecdsa(source, summary):
    lines = []
    for name in ("evppkey_ecdsa.txt", "evppkey_ecdsa_rfc6979.txt"):
        blocks = stanzas(source / name)
        keys = keys_of(blocks)
        for stanza in blocks:
            if provider_only(stanza):
                summary[f"{name}: FIPS-provider-only policy checks"] += 1
                continue
            result = get(stanza, "Result", "ok")
            if get(stanza, "PrivPubKeyPair"):
                mine, theirs = (keys[n] for n in get(stanza, "PrivPubKeyPair").split(":"))
                if mine.kind not in CURVES:
                    summary[f"{name}: {mine.kind} key pair (curve not implemented)"] += 1
                    continue
                xy = point_xy(theirs)
                lines.append(f"keypair {mine.kind} {result} {scalar_hex(mine)} {xy[0]} {xy[1]}")
                continue
            for operation in ("Verify", "DigestVerify", "OneShotDigestVerify", "DigestSign", "Sign"):
                if get(stanza, operation) is not None:
                    break
            else:
                continue
            if operation in ("Verify", "Sign"):
                key = keys[get(stanza, operation)]
                hash_name = (get(stanza, "Ctrl") or "digest:none").partition(":")[2]
            else:
                key = keys[get(stanza, "Key")]
                hash_name = get(stanza, operation)
            hash_name = DIGESTS.get(hash_name.lower(), hash_name.lower())
            if key.kind not in CURVES:
                summary[f"{name}: {operation} with a {key.kind} key (curve or encoding not implemented)"] += 1
                continue
            if operation == "Sign" or (operation == "DigestSign" and (key.kind != "p256" or get(stanza, "NonceType") != "deterministic")):
                summary[f"{name}: {operation} on {key.kind} (only deterministic P-256 signing is implemented)"] += 1
                continue
            if operation == "DigestSign":
                lines.append(f"digestsign {key.kind} {hash_name} {scalar_hex(key)} {input_hex(stanza)} {value_hex(get(stanza, 'Output'))}")
                continue
            xy = point_xy(key)
            kind = "verify" if operation == "Verify" else "digestverify"
            lines.append(f"{kind} {key.kind} {hash_name} {xy[0]} {xy[1]} {input_hex(stanza)} "
                         f"{value_hex(get(stanza, 'Output'))} {result}")
    return lines


def rsa(source, summary):
    lines = []
    for name in ("evppkey_rsa.txt", "evppkey_rsa_common.txt"):
        blocks = stanzas(source / name)
        keys = keys_of(blocks)
        for stanza in blocks:
            operation = next((op for op in ("Verify", "DigestVerify", "OneShotDigestVerify", "VerifyRecover",
                                            "Sign", "DigestSign", "Decrypt", "Encrypt") if get(stanza, op) is not None), None)
            if operation is None:
                continue
            if operation not in ("Verify", "DigestVerify", "OneShotDigestVerify"):
                summary[f"{name}: RSA {operation} (only verification is implemented)"] += 1
                continue
            if provider_only(stanza) or get(stanza, "Availablein") == "legacy":
                summary[f"{name}: FIPS- or legacy-provider-only checks"] += 1
                continue
            controls = {}
            for item in stanza:
                if item[0] in ("Ctrl", "CtrlInit"):
                    key, _, value = item[1].partition(":")
                    controls[key] = value
            key = keys.get(get(stanza, "Key") if operation != "Verify" else get(stanza, "Verify"))
            if key is None:
                summary[f"{name}: verification with a key generated during the test run"] += 1
                continue
            if key.kind not in ("rsa", "rsa-pss"):
                summary[f"{name}: verification with a {key.kind} key"] += 1
                continue
            if key.kind == "rsa-pss":
                summary[f"{name}: RSASSA-PSS-restricted keys (key-embedded parameters are not modelled)"] += 1
                continue
            padding = controls.get("rsa_padding_mode", "pkcs1")
            hash_name = controls.get("digest") or (get(stanza, operation) if operation != "Verify" else None)
            if hash_name is None and padding == "pss":
                hash_name = "sha1"  # OpenSSL's RSA-PSS default digest (RFC 4055)
            if padding not in ("pkcs1", "pss") or hash_name is None:
                summary[f"{name}: RSA {padding} padding{'' if hash_name else ' without a digest'} (not implemented)"] += 1
                continue
            hash_name = DIGESTS.get(hash_name.lower(), hash_name.lower())
            mgf_name = controls.get("rsa_mgf1_md", hash_name).lower()
            mgf = DIGESTS.get(mgf_name, mgf_name)
            salt = controls.get("rsa_pss_saltlen", "auto")
            if salt in ("auto", "max", "auto-digestmax"):
                salt = "auto"
            elif salt == "digest":
                salt = str(hashlib.new(hash_name).digest_size) if hash_name in DIGESTS.values() else "auto"
            n, e = key.public
            lines.append(f"{padding} {n.lstrip(bytes(1)).hex()} {e.lstrip(bytes(1)).hex()} {hash_name} {mgf} {salt} "
                         f"{1 if operation == 'Verify' else 0} {input_hex(stanza)} "
                         f"{value_hex(get(stanza, 'Output'))} {get(stanza, 'Result', 'ok')}")
    return lines


def hexs(value):
    return value.hex() if value else "-"


def x25519(source, summary):
    lines = []
    blocks = stanzas(source / "evppkey_ecx.txt")
    keys = keys_of(blocks)
    for stanza in blocks:
        if provider_only(stanza):
            if get(stanza, "Derive"):
                summary["evppkey_ecx.txt: FIPS-provider-only derive checks"] += 1
            continue
        if get(stanza, "Derive"):
            mine, peer = keys[get(stanza, "Derive")], keys[get(stanza, "PeerKey")]
            if mine.kind != "x25519":
                summary[f"evppkey_ecx.txt: {mine.kind} derive (curve not implemented)"] += 1
                continue
            lines.append(f"derive {get(stanza, 'Result', 'ok')} {hexs(mine.private)} {hexs(peer.public)} {value_hex(get(stanza, 'SharedSecret'))}")
        elif get(stanza, "PrivPubKeyPair"):
            mine, theirs = (keys[n] for n in get(stanza, "PrivPubKeyPair").split(":"))
            if mine.kind != "x25519" or theirs.kind != "x25519":
                summary[f"evppkey_ecx.txt: {mine.kind}/{theirs.kind} key pair (curve not implemented)"] += 1
                continue
            lines.append(f"keypair {get(stanza, 'Result', 'ok')} {hexs(mine.private)} {hexs(theirs.public)}")
        elif any(get(stanza, k) for k in ("OneShotDigestSign", "OneShotDigestVerify", "Sign", "Verify")):
            summary["evppkey_ecx.txt: Ed25519/Ed448 signatures (not implemented)"] += 1
    return lines


def digest(source, summary):
    lines = []
    for stanza in stanzas(source / "evpmd_sha.txt"):
        name = get(stanza, "Digest")
        if name is None:
            continue
        alg = DIGESTS.get(name.lower())
        if alg is None:
            summary[f"evpmd_sha.txt: {name} (hash not implemented)"] += 1
            continue
        chunks = []
        for item in stanza:
            if item[0] == "Input":
                chunks.append([value_hex(item[1]), "1", "1"])
            elif item[0] == "Ncopy":
                chunks[-1][1] = item[1]
            elif item[0] == "Count":
                chunks[-1][2] = item[1]
        lines.append(f"{alg} {value_hex(get(stanza, 'Output'))} " + " ".join(":".join(c) for c in chunks))
    return lines


def hmac(source, summary):
    lines = []
    for stanza in stanzas(source / "evpmac_common.txt"):
        if not (get(stanza, "MAC") or "").startswith("HMAC"):
            if get(stanza, "MAC"):
                summary[f"evpmac_common.txt: {get(stanza, 'MAC')} (MAC not implemented)"] += 1
            continue
        alg = DIGESTS.get((get(stanza, "Algorithm") or "").lower())
        if provider_only(stanza):
            summary["evpmac_common.txt: HMAC FIPS-provider-only policy checks"] += 1
            continue
        if alg is None:
            summary[f"evpmac_common.txt: HMAC-{get(stanza, 'Algorithm')} (hash not implemented)"] += 1
            continue
        lines.append(f"{alg} {value_hex(get(stanza, 'Key'))} {value_hex(get(stanza, 'Input'))} "
                     f"{value_hex(get(stanza, 'Output'))} {get(stanza, 'Result', 'ok')}")
    return lines


def hkdf(source, summary):
    lines = []
    for name, kind in (("evpkdf_hkdf.txt", "KDF"), ("evppkey_kdf_hkdf.txt", "PKEYKDF")):
        for stanza in stanzas(source / name):
            if get(stanza, kind) is None:
                continue
            if provider_only(stanza):
                summary[f"{name}: FIPS-provider-only policy checks"] += 1
                continue
            mode = {"mode:EXTRACT_ONLY": "extract", "mode:EXPAND_ONLY": "expand"}.get(get(stanza, "Ctrl.mode"), "both")
            digest_text = get(stanza, "Ctrl.digest") or get(stanza, "Ctrl.md")
            alg = "absent" if digest_text is None else DIGESTS.get(digest_text.partition(":")[2].lower(), digest_text.partition(":")[2])
            lines.append(f"{mode} {alg} {ctrl_hex(get(stanza, 'Ctrl.IKM'))} {ctrl_hex(get(stanza, 'Ctrl.salt'))} "
                         f"{ctrl_hex(get(stanza, 'Ctrl.info'))} {value_hex(get(stanza, 'Output'))} {get(stanza, 'Result', 'ok')}")
    return lines


def cipher(source, summary):
    lines = []
    for name in ("evpciph_aes_common.txt", "evpciph_chacha.txt"):
        for stanza in stanzas(source / name):
            kind = (get(stanza, "Cipher") or "").lower()
            if not kind:
                continue
            if kind not in ("aes-128-gcm", "aes-192-gcm", "aes-256-gcm", "chacha20", "chacha20-poly1305"):
                summary[f"{name}: {kind} (mode not implemented)"] += 1
                continue
            operation = (get(stanza, "Operation") or "both").lower()
            fields = [value_hex(get(stanza, k)) for k in ("Key", "IV", "AAD", "Plaintext", "Ciphertext", "Tag")]
            lines.append(" ".join([kind, operation, get(stanza, "Result", "ok")] + fields))
    return lines


def main():
    if len(sys.argv) != 2:
        raise SystemExit(__doc__)
    source = Path(sys.argv[1]) / "test/recipes/30-test_evp_data"
    OUT.mkdir(parents=True, exist_ok=True)
    summary = collections.Counter()
    for name, convert in (("digest", digest), ("hmac", hmac), ("hkdf", hkdf), ("cipher", cipher),
                          ("x25519", x25519), ("ecdh", ecdh), ("ecdsa", ecdsa), ("rsa", rsa)):
        lines = convert(source, summary)
        (OUT / f"{name}.txt").write_text("\n".join(lines) + "\n")
        print(f"{name}.txt: {len(lines)} cases")
    (OUT / "SUMMARY.txt").write_text("".join(f"{count} {what}\n" for what, count in sorted(summary.items())))
    used = ["evpmd_sha.txt", "evpmac_common.txt", "evpkdf_hkdf.txt", "evppkey_kdf_hkdf.txt",
            "evpciph_aes_common.txt", "evpciph_chacha.txt", "evppkey_ecx.txt", "evppkey_ecdh.txt",
            "evppkey_ecdsa.txt", "evppkey_ecdsa_rfc6979.txt", "evppkey_rsa.txt", "evppkey_rsa_common.txt"]
    (OUT / "SOURCE-SHA256SUMS").write_text(
        "".join(f"{hashlib.sha256((source / n).read_bytes()).hexdigest()}  {n}\n" for n in used))


if __name__ == "__main__":
    main()