dymokomi/luce-gpu
Source of release 0.5.6
0.5.6 / src / gpu / metal / compute.lucb
#==============================================================================================
#
# compute - Metal compute pipelines and passes
#
# DESCRIPTION:
# A kernel is a compute pipeline state made from its Metal source's `luce_compute`, plus
# the workgroup size Metal takes at each dispatch. A pass is one command buffer: runs of
# dispatches share a serial compute encoder, runs of copies and fills a blit encoder.
# Buffers are hazard-tracked, so each dispatch sees what earlier ones wrote and every
# later command buffer on the queue sees the results; the command buffer retains what it
# binds, so handles may be destroyed once their commands are recorded.
#
#==============================================================================================
## The Metal buffer index of a kernel's push constants (embed_shaders.py moves
## them there, clear of storage buffers 0..15).
let metal_uniform_index: u64 = 30
# MTLLanguageVersion3_0.
let metal_language_3_0: u64 = 3u64 << 16
# MTLArgumentBuffersTier2: texture handles in device buffers.
let metal_argument_buffers_tier2: u64 = 1
# MTLGPUFamilyMetal3: atomic_float on device memory, and buffers' gpuAddress.
let metal_family_metal3: u64 = 5001
struct MetalKernel:
var pipeline: void*?
var group: u32[3]
## Device-level compute state: the built-in fill kernel, made on first use.
struct MetalCompute:
var fill: MetalKernel*?
func metal_limits(device: MetalDevice*) -> Limits:
let native = device.device else return Limits()
let float_atomics = (metal_uint_with(native, metal_sel("supportsFamily:"), metal_family_metal3) & 0xff) != 0
# Every Apple-silicon GPU allows 1024 threads per threadgroup in each
# dimension and in all (the Metal feature set tables); a kernel's own
# pipeline may allow fewer, which Kernel.create checks. Its SIMD groups
# are 32 wide and support every subgroup operation spirv-cross maps to
# simd_* functions.
return Limits(buffer_bytes = metal_uint(native, metal_sel("maxBufferLength")), group_threads = 1024,
group_size = [1024, 1024, 1024], group_count = [0xffffffffu32, 0xffffffffu32, 0xffffffffu32],
shared_bytes = (u32)metal_uint(native, metal_sel("maxThreadgroupMemoryLength")), float_atomic_add = float_atomics,
subgroup_size = 32, subgroup_ops = true, timestamps = true, buffer_addresses = float_atomics,
ray_query = (metal_uint(native, metal_sel("supportsRaytracing")) & 0xff) != 0,
texture_tables = metal_uint(native, metal_sel("argumentBuffersSupport")) == metal_argument_buffers_tier2, texture_table_size = texture_table_limit)
func metal_kernel_create(device: MetalDevice*, msl: c.str, group: u32[3], fast_math: bool, constants: const Constant[]) -> void*!:
let pool = objc_autoreleasePoolPush()
defer objc_autoreleasePoolPop(pool)
let native = metal_required(device.device)
# Kernels are MSL 3.0 (read-write textures, float atomics); without options
# Metal compiles source as an older language version. Metal also defaults
# to fast math, which may drop infinities and NaNs; kernels keep IEEE
# rules, as on Vulkan, unless they ask for fast math.
let options = metal_ptr(try metal_class("MTLCompileOptions"), metal_sel("new")) else error(failed, "could not create Metal compile options")
defer metal_void(options, metal_sel("release"))
metal_uint_arg(options, metal_sel("setLanguageVersion:"), metal_language_3_0)
metal_bool_arg(options, metal_sel("setFastMathEnabled:"), fast_math)
var failure: void*? = none
let library = metal_library(native, metal_sel("newLibraryWithSource:options:error:"), try metal_text(msl), options, &failure) else error(invalid_shader, "could not compile the kernel's Metal source")
defer metal_void(library, metal_sel("release"))
let function = try metal_kernel_function(library, constants)
defer metal_void(function, metal_sel("release"))
let pipeline = metal_pipeline(native, metal_sel("newComputePipelineStateWithFunction:error:"), function, &failure) else error(invalid_shader, "could not build a Metal compute pipeline")
errdefer metal_void(pipeline, metal_sel("release"))
if (u64)group[0] * (u64)group[1] * (u64)group[2] > metal_uint(pipeline, metal_sel("maxTotalThreadsPerThreadgroup")):
error(invalid_shader, "the kernel's workgroup is larger than its Metal pipeline allows")
return (void*)try new MetalKernel(pipeline = pipeline, group = group) in memory.heap
## The library's `luce_compute`, specialized with `constants` (function
## constants at their ids; the others take their defaults). Metal requires
## the specializing call for a function declaring constants even when none
## is given a value, and accepts it for every function (+1).
func metal_kernel_function(library: void*, constants: const Constant[]) -> void*!:
let name = try metal_text("luce_compute")
let values = metal_ptr(try metal_class("MTLFunctionConstantValues"), metal_sel("new")) else error(failed, "could not create Metal function constant values")
defer metal_void(values, metal_sel("release"))
for constant in constants:
var bits = constant.bits
# Bool constants are one byte; MTLDataTypeBool, UInt, Int and Float.
var flag: u8 = (u8)bits
let data: const void* = (const void*)&flag if constant.kind == ConstantKind.boolean else (const void*)&bits
metal_constant_value(values, metal_sel("setConstantValue:type:atIndex:"), data, metal_constant_type(constant.kind), (u64)constant.id)
var failure: void*? = none
return metal_specialized(library, metal_sel("newFunctionWithName:constantValues:error:"), name, values, &failure) else error(invalid_shader, "could not specialize the kernel's Metal function")
func metal_constant_type(kind: ConstantKind) -> u64:
if kind == ConstantKind.boolean: return 53
if kind == ConstantKind.int32: return 29
if kind == ConstantKind.float32: return 3
return 33
extern func metal_constant_value as "objc_msgSend"(object: void*, selector: void*, value: const void*, kind: u64, index: u64)
extern func metal_specialized as "objc_msgSend"(object: void*, selector: void*, name: void*, values: void*, failure: void*?*) -> void*?
func metal_kernel_release(device: void*, native: void*):
_ = device
metal_kernel_destroy((MetalKernel*)native)
func metal_kernel_destroy(kernel: MetalKernel*):
if let pipeline = kernel.pipeline:
metal_void(pipeline, metal_sel("release"))
free(kernel) in memory.heap
func metal_compute_destroy(device: MetalDevice*):
let compute = device.compute else return
if let fill = compute.fill:
metal_kernel_destroy(fill)
free(compute) in memory.heap
device.compute = none
## The built-in kernel writing 32-bit words (shaders/fill.comp).
func metal_fill_kernel(device: MetalDevice*) -> MetalKernel*!:
if device.compute == none:
device.compute = try new MetalCompute() in memory.heap
let compute = metal_required_compute(device)
if let fill = compute.fill:
return fill
var no_constants: Constant[]
let fill = (MetalKernel*)try metal_kernel_create(device, fill_comp_msl, fill_comp_group, fill_comp_fast_math, no_constants)
compute.fill = fill
return fill
func metal_required_compute(device: MetalDevice*) -> MetalCompute*:
return device.compute else trap("Metal compute state is missing")
## The encoder a run of commands records into: dispatches into a compute
## encoder, copies and fills into a blit encoder, one open at a time.
## Which encoder is open: compute, blit or acceleration structure.
let metal_encoder_compute: u8 = 0
let metal_encoder_blit: u8 = 1
let metal_encoder_accel: u8 = 2
struct MetalEncoding:
var command: void*
var encoder: void*?
var kind: u8
## Buffers kernels may reach by address: resident in every compute encoder.
var addressed: BufferState*[]
func compute() -> void*!:
if let encoder = self.encoder:
if self.kind == metal_encoder_compute: return encoder
self.end()
let encoder = metal_ptr(self.command, metal_sel("computeCommandEncoder")) else error(failed, "could not create a Metal compute encoder")
self.encoder = encoder
self.kind = metal_encoder_compute
if self.addressed.length > 0:
let natives = try new void*[self.addressed.length] --- in memory.heap
defer free(natives) in memory.heap
for index in 0..<self.addressed.length:
natives[index] = self.addressed[index].native
# MTLResourceUsageRead | MTLResourceUsageWrite.
metal_use_resources(encoder, metal_sel("useResources:count:usage:"), natives.data, (u64)natives.length, 3)
return encoder
func transfer() -> void*!:
if let encoder = self.encoder:
if self.kind == metal_encoder_blit: return encoder
self.end()
let encoder = metal_ptr(self.command, metal_sel("blitCommandEncoder")) else error(failed, "could not create a Metal blit encoder")
self.encoder = encoder
self.kind = metal_encoder_blit
return encoder
func builder() -> void*!:
if let encoder = self.encoder:
if self.kind == metal_encoder_accel: return encoder
self.end()
let encoder = metal_ptr(self.command, metal_sel("accelerationStructureCommandEncoder")) else error(failed, "could not create a Metal acceleration structure encoder")
self.encoder = encoder
self.kind = metal_encoder_accel
return encoder
func end():
if let encoder = self.encoder:
metal_void(encoder, metal_sel("endEncoding"))
self.encoder = none
## Encode `commands` into command buffers and commit them: one, or one more
## per timestamp, each ending where a timestamp is. The device's submission
## ring keeps them for completion and timing; with timestamps the pass keeps
## them too, for their end times.
func metal_compute_submit(device: MetalDevice*, commands: const ComputeCommand[], stamps: usize, addressed: BufferState*[]) -> ComputeSubmitted!:
let pool = objc_autoreleasePoolPush()
defer objc_autoreleasePoolPop(pool)
# What was encoded before the pass runs before it.
metal_flush(device)
var buffers = try new void*[stamps + 1] --- in memory.heap
var count: usize = 0
errdefer metal_release_buffers(buffers, count)
var command = metal_ptr(metal_required(device.queue), metal_sel("commandBuffer")) else error(failed, "could not create a Metal command buffer")
var encoding = MetalEncoding(command = command, addressed = addressed)
# Scratch memory for the pass's builds, grown as they need it; Metal orders
# builds that share it.
var scratch = MetalScratch()
for index in 0..<commands.length:
let recorded = &commands[index]
if recorded.kind == command_timestamp:
# Everything before the stamp ends with this command buffer: its
# GPU end time is the stamp.
encoding.end()
metal_void(command, metal_sel("retain"))
metal_void(command, metal_sel("commit"))
buffers[count] = command
count += 1
command = metal_ptr(metal_required(device.queue), metal_sel("commandBuffer")) else error(failed, "could not create a Metal command buffer")
encoding = MetalEncoding(command = command, addressed = addressed)
elif recorded.kind == command_copy:
let origin = recorded.buffer else trap("a copy lost its source")
let goal = recorded.target else trap("a copy lost its destination")
metal_copy_buffer(try encoding.transfer(), metal_sel("copyFromBuffer:sourceOffset:toBuffer:destinationOffset:size:"), origin.native, (u64)recorded.offset, goal.native, (u64)recorded.target_offset, (u64)recorded.bytes)
elif recorded.kind == command_fill:
try metal_encode_fill(device, &encoding, recorded)
elif recorded.kind >= command_build_blas:
try metal_encode_build(device, try encoding.builder(), recorded, &scratch)
else:
let kernel = recorded.kernel else trap("a dispatch lost its kernel")
let encoder = try encoding.compute()
try metal_encode_bindings(encoder, try metal_drawing(device), (MetalKernel*)kernel.built(), recorded)
let group = metal_group((MetalKernel*)kernel.built())
if recorded.kind == command_indirect:
let arguments = recorded.buffer else trap("an indirect dispatch lost its arguments")
metal_dispatch_indirect(encoder, metal_sel("dispatchThreadgroupsWithIndirectBuffer:indirectBufferOffset:threadsPerThreadgroup:"), arguments.native, (u64)recorded.offset, group)
else:
metal_dispatch(encoder, metal_sel("dispatchThreadgroups:threadsPerThreadgroup:"),
LuceGpuMetalExtent((u64)recorded.groups[0], (u64)recorded.groups[1], (u64)recorded.groups[2]), group)
encoding.end()
metal_void(command, metal_sel("retain"))
metal_void(command, metal_sel("commit"))
buffers[count] = command
count += 1
var stamped: MetalStamps*? = none
if stamps > 0:
let kept = try new void*[stamps] --- in memory.heap
for index in 0..<stamps:
metal_void(buffers[index], metal_sel("retain"))
kept[index] = buffers[index]
stamped = new MetalStamps(commands = kept) in memory.heap catch failure:
metal_release_buffers(kept, stamps)
error(failure.code, failure.message)
let serial = metal_record_all(device, buffers, count)
var native: void*? = none
if let kept = stamped: native = (void*)kept
return ComputeSubmitted(serial = serial, native = native)
## A pass's timestamps: the retained command buffers each stamp ends.
struct MetalStamps:
var commands: void*[]
func metal_compute_stamps(native: void*?, out: u64[]) -> !:
let stamps = (MetalStamps*)(native else error(invalid_geometry, "the pass recorded no timestamps"))
for index in 0..<out.length:
out[index] = metal_nanoseconds(metal_double(stamps.commands[index], metal_sel("GPUEndTime")))
func metal_compute_release(native: void*):
let stamps = (MetalStamps*)native
metal_release_buffers(stamps.commands, stamps.commands.length)
free(stamps) in memory.heap
func metal_group(kernel: MetalKernel*) -> LuceGpuMetalExtent:
return LuceGpuMetalExtent((u64)kernel.group[0], (u64)kernel.group[1], (u64)kernel.group[2])
## Bind a dispatch's pipeline, buffers, images (a sampled one with its sampler
## at the same index) and its 128 uniform bytes (the block may be padded past
## the bytes given, so Metal sees all of it).
func metal_encode_bindings(encoder: void*, drawing: MetalDrawing*, kernel: MetalKernel*, recorded: const ComputeCommand*) -> !:
metal_object(encoder, metal_sel("setComputePipelineState:"), metal_required(kernel.pipeline))
let program = recorded.kernel else trap("a dispatch lost its kernel")
if let table = recorded.table:
try metal_table_encode(encoder, (MetalTable*)table.native)
for index in 0..<binding_limit:
if let accel = recorded.accels[index]:
try metal_accel_bind(encoder, accel, (u64)index)
for index in 0..<binding_limit:
if program.bindings[index] == (u8)BindingKind.sampler:
metal_bind_texture(encoder, metal_sel("setSamplerState:atIndex:"), metal_sampler_for(drawing, recorded.filters[index]), (u64)index)
if let buffer = recorded.buffers[index]:
metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), buffer.native, 0, (u64)index)
if let texture = recorded.textures[index]:
metal_bind_texture(encoder, metal_sel("setTexture:atIndex:"), metal_required(((MetalTexture*)texture.native).texture), (u64)index)
if program.bindings[index] == (u8)BindingKind.sampled_image:
# Integer textures are never filtered.
let filter = Filter.nearest if texture.format == Format.r32_uint else recorded.filters[index]
metal_bind_texture(encoder, metal_sel("setSamplerState:atIndex:"), metal_sampler_for(drawing, filter), (u64)index)
var uniforms = recorded.uniforms
metal_inline_bytes(encoder, metal_sel("setBytes:length:atIndex:"), &uniforms, (u64)uniform_limit, metal_uniform_index)
## A fill: a blit when the value's four bytes are equal (Metal fills bytes),
## otherwise the built-in word kernel, in runs of at most 2^30 words.
func metal_encode_fill(device: MetalDevice*, encoding: MetalEncoding*, recorded: const ComputeCommand*) -> !:
let target = recorded.buffer else trap("a fill lost its buffer")
let value = recorded.value
let low = (u8)(value & 0xff)
if value == (u32)low * 0x01010101u32:
metal_fill_buffer(try encoding.transfer(), metal_sel("fillBuffer:range:value:"), target.native, LuceGpuMetalRange((u64)recorded.offset, (u64)recorded.bytes), (u64)low)
return
let kernel = try metal_fill_kernel(device)
let encoder = try encoding.compute()
metal_object(encoder, metal_sel("setComputePipelineState:"), metal_required(kernel.pipeline))
let run: usize = 1 << 30
var done: usize = 0
let words = recorded.bytes // 4
while done < words:
let count = run if words - done > run else words - done
metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), target.native, (u64)(recorded.offset + done * 4), 0)
var uniforms: u32[3] = [0, (u32)count, value]
metal_inline_bytes(encoder, metal_sel("setBytes:length:atIndex:"), &uniforms, 12, metal_uniform_index)
metal_dispatch(encoder, metal_sel("dispatchThreadgroups:threadsPerThreadgroup:"), LuceGpuMetalExtent((u64)((count + 63) // 64), 1, 1), metal_group(kernel))
done += count
extern func metal_use_resources as "objc_msgSend"(object: void*, selector: void*, resources: void**, count: u64, usage: u64)
extern func metal_double as "objc_msgSend"(object: void*, selector: void*) -> f64
extern func metal_uint_with as "objc_msgSend"(object: void*, selector: void*, value: u64) -> u64
extern func metal_dispatch as "objc_msgSend"(object: void*, selector: void*, grid: LuceGpuMetalExtent, size: LuceGpuMetalExtent)
extern func metal_dispatch_indirect as "objc_msgSend"(object: void*, selector: void*, arguments: void*, offset: u64, size: LuceGpuMetalExtent)
# mark: Texture tables ========================================================================
## A texture table: a shared buffer of texture handles (MTLResourceID, 8 bytes
## each), which kernels read as an argument buffer at index 29, and the
## textures in it, made resident for each dispatch that reads it.
struct MetalTable:
var buffer: void*?
var handles: u64*
var textures: void*?[]
## The Metal buffer index of a kernel's texture table (embed_shaders.py).
let metal_table_index: u64 = 29
func metal_table_create(device: MetalDevice*, capacity: u32) -> MetalTable*!:
let pool = objc_autoreleasePoolPush()
defer objc_autoreleasePoolPop(pool)
let buffer = metal_new_buffer(metal_required(device.device), metal_sel("newBufferWithLength:options:"), (u64)capacity * 8, metal_storage_shared) else error(out_of_memory, "could not allocate a texture table")
errdefer metal_void(buffer, metal_sel("release"))
let contents = metal_ptr(buffer, metal_sel("contents")) else error(failed, "Metal shared buffer storage is inaccessible")
memory.set(u8[]((u8*)contents, (usize)capacity * 8), 0)
let textures = try new void*?[(usize)capacity] in memory.heap
return try new MetalTable(buffer = buffer, handles = (u64*)contents, textures = textures) in memory.heap
func metal_table_set(table: MetalTable*, index: u32, texture: MetalTexture*):
let native = metal_required(texture.texture)
table.handles[(usize)index] = metal_uint(native, metal_sel("gpuResourceID"))
table.textures[(usize)index] = native
func metal_table_clear(table: MetalTable*, index: u32):
table.handles[(usize)index] = 0
table.textures[(usize)index] = none
func metal_table_destroy(table: MetalTable*):
if let buffer = table.buffer:
metal_void(buffer, metal_sel("release"))
free(table.textures) in memory.heap
free(table) in memory.heap
## Bind a table and make its textures resident for the dispatch.
func metal_table_encode(encoder: void*, table: MetalTable*) -> !:
metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), metal_required(table.buffer), 0, metal_table_index)
let resident = try new void*[table.textures.length] --- in memory.heap
defer free(resident) in memory.heap
var count: usize = 0
for slot in table.textures:
if let texture = slot:
resident[count] = texture
count += 1
if count > 0:
# MTLResourceUsageRead.
metal_use_resources(encoder, metal_sel("useResources:count:usage:"), resident.data, (u64)count, 1)