Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.2

0.5.2 / tools / embed_shaders.py

#!/usr/bin/env python3
"""Embed GLSL shaders for both GPU backends into a Luce Base module.

Each fragment shader becomes SPIR-V words (Vulkan) and Metal Shading Language
source cross-compiled from them, with its entry point named `luce_fragment`;
a vertex shader becomes SPIR-V words only. A compute shader (`.comp`) becomes
`<stem>_comp_words`, `<stem>_comp_msl` (entry `luce_compute`), `<stem>_comp_group`
(its workgroup size), `<stem>_comp_bindings` (what each set-0 binding holds)
`<stem>_comp_fast_math` (whether Metal may relax IEEE float rules) and
`<stem>_comp_table` (whether it reads a texture table), the arguments of
`gpu.Kernel.create`; a kernel with specialization constants
(`layout(constant_id = N) const uint NAME = 1u;`) also gets
`<stem>_comp_constant_ids`, `_kinds`, `_defaults` and `_names`. Builds use the checked-in module, so
glslangValidator and spirv-cross are needed only when a shader changes.

    embed_shaders.py OUTPUT.lucb [--vertex FILE.vert] [-I DIR] [--depfile FILE] SHADER...

A SHADER is `FILE.frag` or `FILE.comp`, or `FILE:STEM:NAME=VALUE,...` to embed
one source under another stem with preprocessor defines, so one kernel can be
embedded several ways: `trace.comp:trace:SPECTRAL=1 trace.comp:trace_rgb:SPECTRAL=0`.
`#include "name.glsl"` resolves beside the shader, then in each `-I` directory,
so packages can share one include (luce-canvas's srgb.glsl) without copies.
`--depfile` writes every source and include the output was made from, as a
Makefile rule, so a build script can re-embed when a shared include changes.
"""
import argparse
import json
from pathlib import Path
import re
import shutil
import struct
import subprocess
import sys
import tempfile

parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('output', type=Path)
parser.add_argument('shaders', nargs='*', help='FILE.frag, FILE.comp, or FILE:STEM:NAME=VALUE,...')
parser.add_argument('--vertex', type=Path, action='append', default=[])
parser.add_argument('-I', '--include', type=Path, action='append', default=[], help='a directory `#include` searches')
parser.add_argument('--depfile', type=Path, help='write the sources and includes the output depends on')
parser.add_argument('--glslang', default='glslangValidator')
parser.add_argument('--spirv-cross', default='spirv-cross')
parser.add_argument('--target-env', default='vulkan1.0', help='the SPIR-V environment of compute shaders (vulkan1.2 for ray queries)')
parser.add_argument('--fast-math', action='append', default=[], metavar='STEM',
                    help='let Metal compile shader or kernel STEM with fast math (no inf/NaN guarantees); repeatable')
parser.add_argument('--public', action='store_true', help='declare the embedded shaders `pub`')
parser.add_argument('--shared', action='store_true', help='the output joins a module that already imports c')
parser.add_argument('--vertex-metal', action='store_true', help='also translate vertex stages to MSL for retained geometry')
args = parser.parse_args()
visibility = 'pub ' if args.public else ''

# Compute kernels see push constants at this Metal buffer index, and a texture
# table (set 1) at the one below, clear of storage buffers at bindings 0..15
# (Metal allows 31 buffer indices).
PUSH_CONSTANT_BUFFER = 30
TABLE_BUFFER = 29
BINDING_LIMIT = 16
# gpu.Kernel's binding kinds.
STORAGE_BUFFER, STORAGE_IMAGE, SAMPLED_IMAGE, ACCELERATION_STRUCTURE, SAMPLER = 1, 2, 3, 4, 5
# gpu.ConstantKind of each specialization constant type a kernel may declare.
CONSTANT_KINDS = {'uint': 1, 'int': 2, 'float': 3, 'bool': 4}


class Shader:
    """One embedding: a source, the stem its names start with, and its defines."""

    def __init__(self, spec):
        path, _, rest = spec.partition(':')
        self.path = Path(path)
        stem, _, defines = rest.partition(':')
        self.stem = stem or self.path.stem
        self.defines = [define for define in defines.split(',') if define]
        if not re.fullmatch(r'[a-z_][a-z0-9_]*', self.stem):
            sys.exit(f'{spec}: a stem is a lowercase Luce name')
        self.stage = {'.frag': 'frag', '.comp': 'comp'}.get(self.path.suffix)
        if self.stage is None:
            sys.exit(f'{spec}: a shader is a .frag or .comp file')


shaders = [Shader(spec) for spec in args.shaders]
for stem in args.fast_math:
    if not any(shader.stem == stem for shader in shaders):
        sys.exit(f'--fast-math {stem}: no shader has that stem')
lines = ['#' + '=' * 94, '#', f'#   {args.output.stem} - Embedded GPU shaders', '#', '#   DESCRIPTION:',
         '#       Generated by luce-gpu/tools/embed_shaders.py from ' + ', '.join(dict.fromkeys(p.name for p in [*args.vertex, *(s.path for s in shaders)])) + '; do not edit.',
         '#', '#' + '=' * 94, ''] + ([] if args.shared else ['import c', ''])
dependencies = set()


def words_of(path, stage, temporary, stem=None, defines=(), environment='vulkan1.0'):
    output = Path(temporary) / ((stem or path.stem) + '.' + stage + '.spv')
    depfile = output.with_suffix('.d')
    includes = [f'-I{directory}' for directory in args.include]
    macros = [f'-D{define}' for define in defines]
    subprocess.run([args.glslang, '-V', '--target-env', environment, *includes, *macros, '--depfile', str(depfile),
                    '-S', stage, str(path), '-o', str(output)], check=True)
    # "out.spv: a.comp b.glsl ...", escaped spaces kept together.
    rule = depfile.read_text(encoding='utf-8').replace('\\\n', ' ')
    for name in re.split(r'(?<!\\)\s+', rule.split(':', 1)[1].strip() if ':' in rule else ''):
        if name:
            dependencies.add(str(Path(name.replace('\\ ', ' ')).resolve()))
    data = output.read_bytes()
    words = struct.unpack(f'<{len(data) // 4}I', data)
    assert words[0] == 0x07230203
    return output, words


def check_msl(name, msl, temporary, standard=None):
    """Compile the Metal source where Metal is (macOS): spirv-cross passes GLSL
    names through that Metal reserves (a helper called `level`), and the device
    would only refuse the shader at run time."""
    if sys.platform != 'darwin' or shutil.which('xcrun') is None:
        return
    source = Path(temporary) / (name + '.metal')
    source.write_text(msl, encoding='utf-8')
    flags = [f'-std={standard}'] if standard else []
    compiled = subprocess.run(['xcrun', '-sdk', 'macosx', 'metal', *flags, '-c', str(source), '-o', str(source.with_suffix('.air'))],
                              capture_output=True, text=True)
    if compiled.returncode != 0:
        sys.exit(f'{name}: the Metal translation does not compile:\n{compiled.stderr}')


def emit_mark(title):
    head = f'# mark: {title} '
    lines.append(head + '=' * (95 - len(head)))
    lines.append('')


def emit_words(name, words):
    global lines
    lines.append(f'{visibility}let {name}: u32[{len(words)}] = [')
    for offset in range(0, len(words), 8):
        lines.append('    ' + ', '.join(f'0x{word:08x}' for word in words[offset:offset + 8]) + ',')
    lines = lines + [']', '']


def emit_text(name, msl):
    escaped = msl.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
    lines.append(f'{visibility}let {name}: c.str = "{escaped}"')


def kernel_layout(shader, spv):
    """The workgroup size and binding kinds of a compute shader, from spirv-cross's
    reflection, refusing what the kernel contract (docs/GPU.md, Compute) leaves out."""
    reflection = json.loads(subprocess.run([args.spirv_cross, str(spv), '--reflect'], check=True, capture_output=True, text=True).stdout)
    entry = reflection['entryPoints'][0]
    if any(entry.get('workgroup_size_is_spec_constant_id', [])):
        sys.exit(f'{shader.path}: give the workgroup size as numbers; embed variants with defines instead')
    for kind in ('ubos', 'subpass_inputs'):
        if reflection.get(kind):
            sys.exit(f'{shader.path}: kernels take storage buffers, images and push constants, not {kind}')
    # A texture table: one runtime array of texture2D at set 1, binding 0.
    table = False
    for resource in reflection.get('separate_images', []):
        if resource.get('set', 0) != 1 or resource.get('binding', 0) != 0 or resource.get('array') != [0] or table:
            sys.exit(f'{shader.path}: a separate image must be the texture table: `layout(set = 1, binding = 0) uniform texture2D name[];`')
        table = True
    bindings = [0] * BINDING_LIMIT
    for kind, code in (('ssbos', STORAGE_BUFFER), ('images', STORAGE_IMAGE), ('textures', SAMPLED_IMAGE), ('acceleration_structures', ACCELERATION_STRUCTURE),
                       ('separate_samplers', SAMPLER)):
        for resource in reflection.get(kind, []):
            binding = resource.get('binding', 0)
            if resource.get('set', 0) != 0 or binding >= BINDING_LIMIT or resource.get('array'):
                sys.exit(f'{shader.path}: {resource["name"]} must be one resource in set 0 at a binding below {BINDING_LIMIT}')
            if bindings[binding]:
                sys.exit(f'{shader.path}: binding {binding} is declared twice')
            bindings[binding] = code
    push = None
    for block in reflection.get('push_constants', []):
        push = reflection['types'][block['type']]['name']
    constants = []
    for constant in reflection.get('specialization_constants', []):
        kind = CONSTANT_KINDS.get(constant['type'])
        if kind is None:
            sys.exit(f'{shader.path}: specialization constant {constant["name"]} is a {constant["type"]}; kernels take uint, int, float and bool')
        constants.append((constant['id'], kind, constant_bits(constant['type'], constant['default_value']), constant['name']))
    return entry['workgroup_size'], bindings, push, table, sorted(constants)


def constant_bits(kind, value):
    """A specialization constant's default as the 32 bits gpu.Constant holds."""
    if kind == 'float':
        return struct.unpack('<I', struct.pack('<f', value))[0]
    if kind == 'bool':
        return 1 if value else 0
    return value & 0xffffffff


def kernel_msl(shader, spv, push, table, temporary):
    # A texture table becomes a Metal argument buffer of texture handles (set 1,
    # in device memory, tier 2); set 0 stays as plain bindings.
    tables = ['--msl-argument-buffers', '--msl-argument-buffer-tier', '1', '--msl-discrete-descriptor-set', '0',
              '--msl-device-argument-buffer', '1', '1'] if table else []
    msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '30000', '--msl-decoration-binding', *tables,
                          '--rename-entry-point', 'main', 'luce_compute', 'comp', str(spv)],
                         check=True, capture_output=True, text=True).stdout
    if 'spvBufferSizeConstants' in msl:
        sys.exit(f'{shader.path}: a kernel cannot ask a buffer for its length (.length()); pass counts in push constants')
    if re.search(r'_atomic \[\[buffer\(', msl):
        sys.exit(f'{shader.path}: image atomics are not supported; use atomics on a storage buffer')
    if push is not None:
        # spirv-cross puts the push-constant block at buffer 0 when it maps
        # bindings directly; move it clear of the storage buffers.
        msl, count = re.subn(r'(constant ' + re.escape(push) + r'& \w+ )\[\[buffer\(\d+\)\]\]', rf'\1[[buffer({PUSH_CONSTANT_BUFFER})]]', msl)
        if count != 1:
            sys.exit(f'{shader.path}: could not place the push-constant block in the Metal translation')
    if table:
        msl, count = re.subn(r'(spvDescriptorSetBuffer1& spvDescriptorSet1 )\[\[buffer\(\d+\)\]\]', rf'\1[[buffer({TABLE_BUFFER})]]', msl)
        if count != 1:
            sys.exit(f'{shader.path}: could not place the texture table in the Metal translation')
    check_msl(shader.stem, msl, temporary, 'metal3.0')
    return msl


with tempfile.TemporaryDirectory(prefix='luce-shader-') as temporary:
    for path in args.vertex:
        emit_mark(f'{path.stem}: vertex stage')
        spv, words = words_of(path, 'vert', temporary)
        emit_words(path.stem + '_vert_words', words)
        if args.vertex_metal:
            msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '20100', '--msl-decoration-binding', '--rename-entry-point', 'main', 'luce_geometry', 'vert', str(spv)], check=True, capture_output=True, text=True).stdout
            check_msl(path.stem, msl, temporary)
            emit_text(f'{path.stem}_vert_msl', msl)
    for shader in shaders:
        if shader.stage == 'frag':
            emit_mark(f'{shader.stem}: fragment stage')
            spv, words = words_of(shader.path, 'frag', temporary, shader.stem, shader.defines)
            emit_words(shader.stem + '_frag_words', words)
            msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '20100', '--msl-decoration-binding',
                                  '--rename-entry-point', 'main', 'luce_fragment', 'frag', str(spv)],
                                 check=True, capture_output=True, text=True).stdout
            check_msl(shader.stem, msl, temporary)
            emit_text(f'{shader.stem}_frag_msl', msl)
            lines.append(f'{visibility}let {shader.stem}_frag_fast_math: bool = {"true" if shader.stem in args.fast_math else "false"}')
            lines.append('')
            continue
        emit_mark(f'{shader.stem}: compute kernel' + (f' ({", ".join(shader.defines)})' if shader.defines else ''))
        spv, words = words_of(shader.path, 'comp', temporary, shader.stem, shader.defines, args.target_env)
        group, bindings, push, table, constants = kernel_layout(shader, spv)
        emit_words(shader.stem + '_comp_words', words)
        emit_text(f'{shader.stem}_comp_msl', kernel_msl(shader, spv, push, table, temporary))
        lines.append(f'{visibility}let {shader.stem}_comp_group: u32[3] = [{", ".join(str(size) for size in group)}]')
        lines.append(f'{visibility}let {shader.stem}_comp_bindings: u8[{BINDING_LIMIT}] = [{", ".join(str(kind) for kind in bindings)}]')
        lines.append(f'{visibility}let {shader.stem}_comp_fast_math: bool = {"true" if shader.stem in args.fast_math else "false"}')
        lines.append(f'{visibility}let {shader.stem}_comp_table: bool = {"true" if table else "false"}')
        if constants:
            # The specialization constants, by id: kind (gpu.ConstantKind), default bits, name.
            count = len(constants)
            lines.append(f'{visibility}let {shader.stem}_comp_constant_ids: u32[{count}] = [{", ".join(str(c[0]) for c in constants)}]')
            lines.append(f'{visibility}let {shader.stem}_comp_constant_kinds: u8[{count}] = [{", ".join(str(c[1]) for c in constants)}]')
            lines.append(f'{visibility}let {shader.stem}_comp_constant_defaults: u32[{count}] = [{", ".join(f"0x{c[2]:08x}" for c in constants)}]')
            lines.append(f'{visibility}let {shader.stem}_comp_constant_names: c.str[{count}] = [{", ".join(chr(34) + c[3] + chr(34) for c in constants)}]')
        lines.append('')
args.output.write_text('\n'.join(lines).rstrip('\n') + '\n', encoding='utf-8')
if args.depfile:
    names = ' '.join(sorted(name.replace(' ', '\\ ') for name in dependencies))
    args.depfile.write_text(f'{args.output}: {names}\n', encoding='utf-8')
print(f'wrote {args.output}')