dymokomi/luce-gpu
Source of release 0.5.4
0.5.4 / tools / embed_shaders.py
#!/usr/bin/env python3
"""Embed GLSL shaders for both GPU backends into a Luce Base module.
Each fragment shader becomes SPIR-V words (Vulkan) and Metal Shading Language
source cross-compiled from them, with its entry point named `luce_fragment`;
a vertex shader becomes SPIR-V words only. A compute shader (`.comp`) becomes
`<stem>_comp_words`, `<stem>_comp_msl` (entry `luce_compute`), `<stem>_comp_group`
(its workgroup size), `<stem>_comp_bindings` (what each set-0 binding holds)
`<stem>_comp_fast_math` (whether Metal may relax IEEE float rules) and
`<stem>_comp_table` (whether it reads a texture table), the arguments of
`gpu.Kernel.create`; a kernel with specialization constants
(`layout(constant_id = N) const uint NAME = 1u;`) also gets
`<stem>_comp_constant_ids`, `_kinds`, `_defaults` and `_names`. Builds use the checked-in module, so
glslangValidator and spirv-cross are needed only when a shader changes.
embed_shaders.py OUTPUT.lucb [--vertex FILE.vert] [-I DIR] [--depfile FILE] SHADER...
A SHADER is `FILE.frag` or `FILE.comp`, or `FILE:STEM:NAME=VALUE,...` to embed
one source under another stem with preprocessor defines, so one kernel can be
embedded several ways: `trace.comp:trace:SPECTRAL=1 trace.comp:trace_rgb:SPECTRAL=0`.
`#include "name.glsl"` resolves beside the shader, then in each `-I` directory,
so packages can share one include (luce-canvas's srgb.glsl) without copies.
`--depfile` writes every source and include the output was made from, as a
Makefile rule, so a build script can re-embed when a shared include changes.
"""
import argparse
import json
from pathlib import Path
import re
import shutil
import struct
import subprocess
import sys
import tempfile
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('output', type=Path)
parser.add_argument('shaders', nargs='*', help='FILE.frag, FILE.comp, or FILE:STEM:NAME=VALUE,...')
parser.add_argument('--vertex', type=Path, action='append', default=[])
parser.add_argument('-I', '--include', type=Path, action='append', default=[], help='a directory `#include` searches')
parser.add_argument('--depfile', type=Path, help='write the sources and includes the output depends on')
parser.add_argument('--glslang', default='glslangValidator')
parser.add_argument('--spirv-cross', default='spirv-cross')
parser.add_argument('--target-env', default='vulkan1.0', help='the SPIR-V environment of compute shaders (vulkan1.2 for ray queries)')
parser.add_argument('--fast-math', action='append', default=[], metavar='STEM',
help='let Metal compile shader or kernel STEM with fast math (no inf/NaN guarantees); repeatable')
parser.add_argument('--public', action='store_true', help='declare the embedded shaders `pub`')
parser.add_argument('--shared', action='store_true', help='the output joins a module that already imports c')
parser.add_argument('--vertex-metal', action='store_true', help='also translate vertex stages to MSL for retained geometry')
args = parser.parse_args()
visibility = 'pub ' if args.public else ''
# Compute kernels see push constants at this Metal buffer index, and a texture
# table (set 1) at the one below, clear of storage buffers at bindings 0..15
# (Metal allows 31 buffer indices).
PUSH_CONSTANT_BUFFER = 30
TABLE_BUFFER = 29
BINDING_LIMIT = 16
# gpu.Kernel's binding kinds.
STORAGE_BUFFER, STORAGE_IMAGE, SAMPLED_IMAGE, ACCELERATION_STRUCTURE, SAMPLER = 1, 2, 3, 4, 5
# gpu.ConstantKind of each specialization constant type a kernel may declare.
CONSTANT_KINDS = {'uint': 1, 'int': 2, 'float': 3, 'bool': 4}
class Shader:
"""One embedding: a source, the stem its names start with, and its defines."""
def __init__(self, spec):
path, _, rest = spec.partition(':')
self.path = Path(path)
stem, _, defines = rest.partition(':')
self.stem = stem or self.path.stem
self.defines = [define for define in defines.split(',') if define]
if not re.fullmatch(r'[a-z_][a-z0-9_]*', self.stem):
sys.exit(f'{spec}: a stem is a lowercase Luce name')
self.stage = {'.frag': 'frag', '.comp': 'comp'}.get(self.path.suffix)
if self.stage is None:
sys.exit(f'{spec}: a shader is a .frag or .comp file')
shaders = [Shader(spec) for spec in args.shaders]
for stem in args.fast_math:
if not any(shader.stem == stem for shader in shaders):
sys.exit(f'--fast-math {stem}: no shader has that stem')
lines = ['#' + '=' * 94, '#', f'# {args.output.stem} - Embedded GPU shaders', '#', '# DESCRIPTION:',
'# Generated by luce-gpu/tools/embed_shaders.py from ' + ', '.join(dict.fromkeys(p.name for p in [*args.vertex, *(s.path for s in shaders)])) + '; do not edit.',
'#', '#' + '=' * 94, ''] + ([] if args.shared else ['import c', ''])
dependencies = set()
def words_of(path, stage, temporary, stem=None, defines=(), environment='vulkan1.0'):
output = Path(temporary) / ((stem or path.stem) + '.' + stage + '.spv')
depfile = output.with_suffix('.d')
includes = [f'-I{directory}' for directory in args.include]
macros = [f'-D{define}' for define in defines]
subprocess.run([args.glslang, '-V', '--target-env', environment, *includes, *macros, '--depfile', str(depfile),
'-S', stage, str(path), '-o', str(output)], check=True)
# "out.spv: a.comp b.glsl ...", escaped spaces kept together.
rule = depfile.read_text(encoding='utf-8').replace('\\\n', ' ')
for name in re.split(r'(?<!\\)\s+', rule.split(':', 1)[1].strip() if ':' in rule else ''):
if name:
dependencies.add(str(Path(name.replace('\\ ', ' ')).resolve()))
data = output.read_bytes()
words = struct.unpack(f'<{len(data) // 4}I', data)
assert words[0] == 0x07230203
return output, words
def check_msl(name, msl, temporary, standard=None):
"""Compile the Metal source where Metal is (macOS): spirv-cross passes GLSL
names through that Metal reserves (a helper called `level`), and the device
would only refuse the shader at run time."""
if sys.platform != 'darwin' or shutil.which('xcrun') is None:
return
source = Path(temporary) / (name + '.metal')
source.write_text(msl, encoding='utf-8')
flags = [f'-std={standard}'] if standard else []
compiled = subprocess.run(['xcrun', '-sdk', 'macosx', 'metal', *flags, '-c', str(source), '-o', str(source.with_suffix('.air'))],
capture_output=True, text=True)
if compiled.returncode != 0:
sys.exit(f'{name}: the Metal translation does not compile:\n{compiled.stderr}')
def emit_mark(title):
head = f'# mark: {title} '
lines.append(head + '=' * (95 - len(head)))
lines.append('')
def emit_words(name, words):
global lines
lines.append(f'{visibility}let {name}: u32[{len(words)}] = [')
for offset in range(0, len(words), 8):
lines.append(' ' + ', '.join(f'0x{word:08x}' for word in words[offset:offset + 8]) + ',')
lines = lines + [']', '']
def emit_text(name, msl):
escaped = msl.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
lines.append(f'{visibility}let {name}: c.str = "{escaped}"')
def kernel_layout(shader, spv):
"""The workgroup size and binding kinds of a compute shader, from spirv-cross's
reflection, refusing what the kernel contract (docs/GPU.md, Compute) leaves out."""
reflection = json.loads(subprocess.run([args.spirv_cross, str(spv), '--reflect'], check=True, capture_output=True, text=True).stdout)
entry = reflection['entryPoints'][0]
if any(entry.get('workgroup_size_is_spec_constant_id', [])):
sys.exit(f'{shader.path}: give the workgroup size as numbers; embed variants with defines instead')
for kind in ('ubos', 'subpass_inputs'):
if reflection.get(kind):
sys.exit(f'{shader.path}: kernels take storage buffers, images and push constants, not {kind}')
# A texture table: one runtime array of texture2D at set 1, binding 0.
table = False
for resource in reflection.get('separate_images', []):
if resource.get('set', 0) != 1 or resource.get('binding', 0) != 0 or resource.get('array') != [0] or table:
sys.exit(f'{shader.path}: a separate image must be the texture table: `layout(set = 1, binding = 0) uniform texture2D name[];`')
table = True
bindings = [0] * BINDING_LIMIT
for kind, code in (('ssbos', STORAGE_BUFFER), ('images', STORAGE_IMAGE), ('textures', SAMPLED_IMAGE), ('acceleration_structures', ACCELERATION_STRUCTURE),
('separate_samplers', SAMPLER)):
for resource in reflection.get(kind, []):
binding = resource.get('binding', 0)
if resource.get('set', 0) != 0 or binding >= BINDING_LIMIT or resource.get('array'):
sys.exit(f'{shader.path}: {resource["name"]} must be one resource in set 0 at a binding below {BINDING_LIMIT}')
if bindings[binding]:
sys.exit(f'{shader.path}: binding {binding} is declared twice')
bindings[binding] = code
push = None
for block in reflection.get('push_constants', []):
push = reflection['types'][block['type']]['name']
constants = []
for constant in reflection.get('specialization_constants', []):
kind = CONSTANT_KINDS.get(constant['type'])
if kind is None:
sys.exit(f'{shader.path}: specialization constant {constant["name"]} is a {constant["type"]}; kernels take uint, int, float and bool')
constants.append((constant['id'], kind, constant_bits(constant['type'], constant['default_value']), constant['name']))
return entry['workgroup_size'], bindings, push, table, sorted(constants)
def constant_bits(kind, value):
"""A specialization constant's default as the 32 bits gpu.Constant holds."""
if kind == 'float':
return struct.unpack('<I', struct.pack('<f', value))[0]
if kind == 'bool':
return 1 if value else 0
return value & 0xffffffff
def kernel_msl(shader, spv, push, table, temporary):
# A texture table becomes a Metal argument buffer of texture handles (set 1,
# in device memory, tier 2); set 0 stays as plain bindings.
tables = ['--msl-argument-buffers', '--msl-argument-buffer-tier', '1', '--msl-discrete-descriptor-set', '0',
'--msl-device-argument-buffer', '1', '1'] if table else []
msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '30000', '--msl-decoration-binding', *tables,
'--rename-entry-point', 'main', 'luce_compute', 'comp', str(spv)],
check=True, capture_output=True, text=True).stdout
if 'spvBufferSizeConstants' in msl:
sys.exit(f'{shader.path}: a kernel cannot ask a buffer for its length (.length()); pass counts in push constants')
if re.search(r'_atomic \[\[buffer\(', msl):
sys.exit(f'{shader.path}: image atomics are not supported; use atomics on a storage buffer')
if push is not None:
# spirv-cross puts the push-constant block at buffer 0 when it maps
# bindings directly; move it clear of the storage buffers.
msl, count = re.subn(r'(constant ' + re.escape(push) + r'& \w+ )\[\[buffer\(\d+\)\]\]', rf'\1[[buffer({PUSH_CONSTANT_BUFFER})]]', msl)
if count != 1:
sys.exit(f'{shader.path}: could not place the push-constant block in the Metal translation')
if table:
msl, count = re.subn(r'(spvDescriptorSetBuffer1& spvDescriptorSet1 )\[\[buffer\(\d+\)\]\]', rf'\1[[buffer({TABLE_BUFFER})]]', msl)
if count != 1:
sys.exit(f'{shader.path}: could not place the texture table in the Metal translation')
check_msl(shader.stem, msl, temporary, 'metal3.0')
return msl
with tempfile.TemporaryDirectory(prefix='luce-shader-') as temporary:
for path in args.vertex:
emit_mark(f'{path.stem}: vertex stage')
spv, words = words_of(path, 'vert', temporary)
emit_words(path.stem + '_vert_words', words)
if args.vertex_metal:
msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '20100', '--msl-decoration-binding', '--rename-entry-point', 'main', 'luce_geometry', 'vert', str(spv)], check=True, capture_output=True, text=True).stdout
check_msl(path.stem, msl, temporary)
emit_text(f'{path.stem}_vert_msl', msl)
for shader in shaders:
if shader.stage == 'frag':
emit_mark(f'{shader.stem}: fragment stage')
spv, words = words_of(shader.path, 'frag', temporary, shader.stem, shader.defines)
emit_words(shader.stem + '_frag_words', words)
msl = subprocess.run([args.spirv_cross, '--msl', '--msl-version', '20100', '--msl-decoration-binding',
'--rename-entry-point', 'main', 'luce_fragment', 'frag', str(spv)],
check=True, capture_output=True, text=True).stdout
check_msl(shader.stem, msl, temporary)
emit_text(f'{shader.stem}_frag_msl', msl)
lines.append(f'{visibility}let {shader.stem}_frag_fast_math: bool = {"true" if shader.stem in args.fast_math else "false"}')
lines.append('')
continue
emit_mark(f'{shader.stem}: compute kernel' + (f' ({", ".join(shader.defines)})' if shader.defines else ''))
spv, words = words_of(shader.path, 'comp', temporary, shader.stem, shader.defines, args.target_env)
group, bindings, push, table, constants = kernel_layout(shader, spv)
emit_words(shader.stem + '_comp_words', words)
emit_text(f'{shader.stem}_comp_msl', kernel_msl(shader, spv, push, table, temporary))
lines.append(f'{visibility}let {shader.stem}_comp_group: u32[3] = [{", ".join(str(size) for size in group)}]')
lines.append(f'{visibility}let {shader.stem}_comp_bindings: u8[{BINDING_LIMIT}] = [{", ".join(str(kind) for kind in bindings)}]')
lines.append(f'{visibility}let {shader.stem}_comp_fast_math: bool = {"true" if shader.stem in args.fast_math else "false"}')
lines.append(f'{visibility}let {shader.stem}_comp_table: bool = {"true" if table else "false"}')
if constants:
# The specialization constants, by id: kind (gpu.ConstantKind), default bits, name.
count = len(constants)
lines.append(f'{visibility}let {shader.stem}_comp_constant_ids: u32[{count}] = [{", ".join(str(c[0]) for c in constants)}]')
lines.append(f'{visibility}let {shader.stem}_comp_constant_kinds: u8[{count}] = [{", ".join(str(c[1]) for c in constants)}]')
lines.append(f'{visibility}let {shader.stem}_comp_constant_defaults: u32[{count}] = [{", ".join(f"0x{c[2]:08x}" for c in constants)}]')
lines.append(f'{visibility}let {shader.stem}_comp_constant_names: c.str[{count}] = [{", ".join(chr(34) + c[3] + chr(34) for c in constants)}]')
lines.append('')
args.output.write_text('\n'.join(lines).rstrip('\n') + '\n', encoding='utf-8')
if args.depfile:
names = ' '.join(sorted(name.replace(' ', '\\ ') for name in dependencies))
args.depfile.write_text(f'{args.output}: {names}\n', encoding='utf-8')
print(f'wrote {args.output}')