Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.6

0.5.6 / src / gpu / metal / compute.lucb

#==============================================================================================
#
#   compute - Metal compute pipelines and passes
#
#   DESCRIPTION:
#       A kernel is a compute pipeline state made from its Metal source's `luce_compute`, plus
#       the workgroup size Metal takes at each dispatch. A pass is one command buffer: runs of
#       dispatches share a serial compute encoder, runs of copies and fills a blit encoder.
#       Buffers are hazard-tracked, so each dispatch sees what earlier ones wrote and every
#       later command buffer on the queue sees the results; the command buffer retains what it
#       binds, so handles may be destroyed once their commands are recorded.
#
#==============================================================================================

## The Metal buffer index of a kernel's push constants (embed_shaders.py moves
## them there, clear of storage buffers 0..15).
let metal_uniform_index: u64 = 30
# MTLLanguageVersion3_0.
let metal_language_3_0: u64 = 3u64 << 16
# MTLArgumentBuffersTier2: texture handles in device buffers.
let metal_argument_buffers_tier2: u64 = 1
# MTLGPUFamilyMetal3: atomic_float on device memory, and buffers' gpuAddress.
let metal_family_metal3: u64 = 5001

struct MetalKernel:
    var pipeline: void*?
    var group: u32[3]

## Device-level compute state: the built-in fill kernel, made on first use.
struct MetalCompute:
    var fill: MetalKernel*?

func metal_limits(device: MetalDevice*) -> Limits:
    let native = device.device else return Limits()
    let float_atomics = (metal_uint_with(native, metal_sel("supportsFamily:"), metal_family_metal3) & 0xff) != 0
    # Every Apple-silicon GPU allows 1024 threads per threadgroup in each
    # dimension and in all (the Metal feature set tables); a kernel's own
    # pipeline may allow fewer, which Kernel.create checks. Its SIMD groups
    # are 32 wide and support every subgroup operation spirv-cross maps to
    # simd_* functions.
    return Limits(buffer_bytes = metal_uint(native, metal_sel("maxBufferLength")), group_threads = 1024,
                  group_size = [1024, 1024, 1024], group_count = [0xffffffffu32, 0xffffffffu32, 0xffffffffu32],
                  shared_bytes = (u32)metal_uint(native, metal_sel("maxThreadgroupMemoryLength")), float_atomic_add = float_atomics,
                  subgroup_size = 32, subgroup_ops = true, timestamps = true, buffer_addresses = float_atomics,
                  ray_query = (metal_uint(native, metal_sel("supportsRaytracing")) & 0xff) != 0,
                  texture_tables = metal_uint(native, metal_sel("argumentBuffersSupport")) == metal_argument_buffers_tier2, texture_table_size = texture_table_limit)

func metal_kernel_create(device: MetalDevice*, msl: c.str, group: u32[3], fast_math: bool, constants: const Constant[]) -> void*!:
    let pool = objc_autoreleasePoolPush()
    defer objc_autoreleasePoolPop(pool)
    let native = metal_required(device.device)
    # Kernels are MSL 3.0 (read-write textures, float atomics); without options
    # Metal compiles source as an older language version. Metal also defaults
    # to fast math, which may drop infinities and NaNs; kernels keep IEEE
    # rules, as on Vulkan, unless they ask for fast math.
    let options = metal_ptr(try metal_class("MTLCompileOptions"), metal_sel("new")) else error(failed, "could not create Metal compile options")
    defer metal_void(options, metal_sel("release"))
    metal_uint_arg(options, metal_sel("setLanguageVersion:"), metal_language_3_0)
    metal_bool_arg(options, metal_sel("setFastMathEnabled:"), fast_math)
    var failure: void*? = none
    let library = metal_library(native, metal_sel("newLibraryWithSource:options:error:"), try metal_text(msl), options, &failure) else error(invalid_shader, "could not compile the kernel's Metal source")
    defer metal_void(library, metal_sel("release"))
    let function = try metal_kernel_function(library, constants)
    defer metal_void(function, metal_sel("release"))
    let pipeline = metal_pipeline(native, metal_sel("newComputePipelineStateWithFunction:error:"), function, &failure) else error(invalid_shader, "could not build a Metal compute pipeline")
    errdefer metal_void(pipeline, metal_sel("release"))
    if (u64)group[0] * (u64)group[1] * (u64)group[2] > metal_uint(pipeline, metal_sel("maxTotalThreadsPerThreadgroup")):
        error(invalid_shader, "the kernel's workgroup is larger than its Metal pipeline allows")
    return (void*)try new MetalKernel(pipeline = pipeline, group = group) in memory.heap

## The library's `luce_compute`, specialized with `constants` (function
## constants at their ids; the others take their defaults). Metal requires
## the specializing call for a function declaring constants even when none
## is given a value, and accepts it for every function (+1).
func metal_kernel_function(library: void*, constants: const Constant[]) -> void*!:
    let name = try metal_text("luce_compute")
    let values = metal_ptr(try metal_class("MTLFunctionConstantValues"), metal_sel("new")) else error(failed, "could not create Metal function constant values")
    defer metal_void(values, metal_sel("release"))
    for constant in constants:
        var bits = constant.bits
        # Bool constants are one byte; MTLDataTypeBool, UInt, Int and Float.
        var flag: u8 = (u8)bits
        let data: const void* = (const void*)&flag if constant.kind == ConstantKind.boolean else (const void*)&bits
        metal_constant_value(values, metal_sel("setConstantValue:type:atIndex:"), data, metal_constant_type(constant.kind), (u64)constant.id)
    var failure: void*? = none
    return metal_specialized(library, metal_sel("newFunctionWithName:constantValues:error:"), name, values, &failure) else error(invalid_shader, "could not specialize the kernel's Metal function")

func metal_constant_type(kind: ConstantKind) -> u64:
    if kind == ConstantKind.boolean: return 53
    if kind == ConstantKind.int32: return 29
    if kind == ConstantKind.float32: return 3
    return 33

extern func metal_constant_value as "objc_msgSend"(object: void*, selector: void*, value: const void*, kind: u64, index: u64)
extern func metal_specialized as "objc_msgSend"(object: void*, selector: void*, name: void*, values: void*, failure: void*?*) -> void*?

func metal_kernel_release(device: void*, native: void*):
    _ = device
    metal_kernel_destroy((MetalKernel*)native)

func metal_kernel_destroy(kernel: MetalKernel*):
    if let pipeline = kernel.pipeline:
        metal_void(pipeline, metal_sel("release"))
    free(kernel) in memory.heap

func metal_compute_destroy(device: MetalDevice*):
    let compute = device.compute else return
    if let fill = compute.fill:
        metal_kernel_destroy(fill)
    free(compute) in memory.heap
    device.compute = none

## The built-in kernel writing 32-bit words (shaders/fill.comp).
func metal_fill_kernel(device: MetalDevice*) -> MetalKernel*!:
    if device.compute == none:
        device.compute = try new MetalCompute() in memory.heap
    let compute = metal_required_compute(device)
    if let fill = compute.fill:
        return fill
    var no_constants: Constant[]
    let fill = (MetalKernel*)try metal_kernel_create(device, fill_comp_msl, fill_comp_group, fill_comp_fast_math, no_constants)
    compute.fill = fill
    return fill

func metal_required_compute(device: MetalDevice*) -> MetalCompute*:
    return device.compute else trap("Metal compute state is missing")

## The encoder a run of commands records into: dispatches into a compute
## encoder, copies and fills into a blit encoder, one open at a time.
## Which encoder is open: compute, blit or acceleration structure.
let metal_encoder_compute: u8 = 0
let metal_encoder_blit: u8 = 1
let metal_encoder_accel: u8 = 2

struct MetalEncoding:
    var command: void*
    var encoder: void*?
    var kind: u8
    ## Buffers kernels may reach by address: resident in every compute encoder.
    var addressed: BufferState*[]

    func compute() -> void*!:
        if let encoder = self.encoder:
            if self.kind == metal_encoder_compute: return encoder
            self.end()
        let encoder = metal_ptr(self.command, metal_sel("computeCommandEncoder")) else error(failed, "could not create a Metal compute encoder")
        self.encoder = encoder
        self.kind = metal_encoder_compute
        if self.addressed.length > 0:
            let natives = try new void*[self.addressed.length] --- in memory.heap
            defer free(natives) in memory.heap
            for index in 0..<self.addressed.length:
                natives[index] = self.addressed[index].native
            # MTLResourceUsageRead | MTLResourceUsageWrite.
            metal_use_resources(encoder, metal_sel("useResources:count:usage:"), natives.data, (u64)natives.length, 3)
        return encoder

    func transfer() -> void*!:
        if let encoder = self.encoder:
            if self.kind == metal_encoder_blit: return encoder
            self.end()
        let encoder = metal_ptr(self.command, metal_sel("blitCommandEncoder")) else error(failed, "could not create a Metal blit encoder")
        self.encoder = encoder
        self.kind = metal_encoder_blit
        return encoder

    func builder() -> void*!:
        if let encoder = self.encoder:
            if self.kind == metal_encoder_accel: return encoder
            self.end()
        let encoder = metal_ptr(self.command, metal_sel("accelerationStructureCommandEncoder")) else error(failed, "could not create a Metal acceleration structure encoder")
        self.encoder = encoder
        self.kind = metal_encoder_accel
        return encoder

    func end():
        if let encoder = self.encoder:
            metal_void(encoder, metal_sel("endEncoding"))
        self.encoder = none

## Encode `commands` into command buffers and commit them: one, or one more
## per timestamp, each ending where a timestamp is. The device's submission
## ring keeps them for completion and timing; with timestamps the pass keeps
## them too, for their end times.
func metal_compute_submit(device: MetalDevice*, commands: const ComputeCommand[], stamps: usize, addressed: BufferState*[]) -> ComputeSubmitted!:
    let pool = objc_autoreleasePoolPush()
    defer objc_autoreleasePoolPop(pool)
    # What was encoded before the pass runs before it.
    metal_flush(device)
    var buffers = try new void*[stamps + 1] --- in memory.heap
    var count: usize = 0
    errdefer metal_release_buffers(buffers, count)
    var command = metal_ptr(metal_required(device.queue), metal_sel("commandBuffer")) else error(failed, "could not create a Metal command buffer")
    var encoding = MetalEncoding(command = command, addressed = addressed)
    # Scratch memory for the pass's builds, grown as they need it; Metal orders
    # builds that share it.
    var scratch = MetalScratch()
    for index in 0..<commands.length:
        let recorded = &commands[index]
        if recorded.kind == command_timestamp:
            # Everything before the stamp ends with this command buffer: its
            # GPU end time is the stamp.
            encoding.end()
            metal_void(command, metal_sel("retain"))
            metal_void(command, metal_sel("commit"))
            buffers[count] = command
            count += 1
            command = metal_ptr(metal_required(device.queue), metal_sel("commandBuffer")) else error(failed, "could not create a Metal command buffer")
            encoding = MetalEncoding(command = command, addressed = addressed)
        elif recorded.kind == command_copy:
            let origin = recorded.buffer else trap("a copy lost its source")
            let goal = recorded.target else trap("a copy lost its destination")
            metal_copy_buffer(try encoding.transfer(), metal_sel("copyFromBuffer:sourceOffset:toBuffer:destinationOffset:size:"), origin.native, (u64)recorded.offset, goal.native, (u64)recorded.target_offset, (u64)recorded.bytes)
        elif recorded.kind == command_fill:
            try metal_encode_fill(device, &encoding, recorded)
        elif recorded.kind >= command_build_blas:
            try metal_encode_build(device, try encoding.builder(), recorded, &scratch)
        else:
            let kernel = recorded.kernel else trap("a dispatch lost its kernel")
            let encoder = try encoding.compute()
            try metal_encode_bindings(encoder, try metal_drawing(device), (MetalKernel*)kernel.built(), recorded)
            let group = metal_group((MetalKernel*)kernel.built())
            if recorded.kind == command_indirect:
                let arguments = recorded.buffer else trap("an indirect dispatch lost its arguments")
                metal_dispatch_indirect(encoder, metal_sel("dispatchThreadgroupsWithIndirectBuffer:indirectBufferOffset:threadsPerThreadgroup:"), arguments.native, (u64)recorded.offset, group)
            else:
                metal_dispatch(encoder, metal_sel("dispatchThreadgroups:threadsPerThreadgroup:"),
                               LuceGpuMetalExtent((u64)recorded.groups[0], (u64)recorded.groups[1], (u64)recorded.groups[2]), group)
    encoding.end()
    metal_void(command, metal_sel("retain"))
    metal_void(command, metal_sel("commit"))
    buffers[count] = command
    count += 1
    var stamped: MetalStamps*? = none
    if stamps > 0:
        let kept = try new void*[stamps] --- in memory.heap
        for index in 0..<stamps:
            metal_void(buffers[index], metal_sel("retain"))
            kept[index] = buffers[index]
        stamped = new MetalStamps(commands = kept) in memory.heap catch failure:
            metal_release_buffers(kept, stamps)
            error(failure.code, failure.message)
    let serial = metal_record_all(device, buffers, count)
    var native: void*? = none
    if let kept = stamped: native = (void*)kept
    return ComputeSubmitted(serial = serial, native = native)

## A pass's timestamps: the retained command buffers each stamp ends.
struct MetalStamps:
    var commands: void*[]

func metal_compute_stamps(native: void*?, out: u64[]) -> !:
    let stamps = (MetalStamps*)(native else error(invalid_geometry, "the pass recorded no timestamps"))
    for index in 0..<out.length:
        out[index] = metal_nanoseconds(metal_double(stamps.commands[index], metal_sel("GPUEndTime")))

func metal_compute_release(native: void*):
    let stamps = (MetalStamps*)native
    metal_release_buffers(stamps.commands, stamps.commands.length)
    free(stamps) in memory.heap

func metal_group(kernel: MetalKernel*) -> LuceGpuMetalExtent:
    return LuceGpuMetalExtent((u64)kernel.group[0], (u64)kernel.group[1], (u64)kernel.group[2])

## Bind a dispatch's pipeline, buffers, images (a sampled one with its sampler
## at the same index) and its 128 uniform bytes (the block may be padded past
## the bytes given, so Metal sees all of it).
func metal_encode_bindings(encoder: void*, drawing: MetalDrawing*, kernel: MetalKernel*, recorded: const ComputeCommand*) -> !:
    metal_object(encoder, metal_sel("setComputePipelineState:"), metal_required(kernel.pipeline))
    let program = recorded.kernel else trap("a dispatch lost its kernel")
    if let table = recorded.table:
        try metal_table_encode(encoder, (MetalTable*)table.native)
    for index in 0..<binding_limit:
        if let accel = recorded.accels[index]:
            try metal_accel_bind(encoder, accel, (u64)index)
    for index in 0..<binding_limit:
        if program.bindings[index] == (u8)BindingKind.sampler:
            metal_bind_texture(encoder, metal_sel("setSamplerState:atIndex:"), metal_sampler_for(drawing, recorded.filters[index]), (u64)index)
        if let buffer = recorded.buffers[index]:
            metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), buffer.native, 0, (u64)index)
        if let texture = recorded.textures[index]:
            metal_bind_texture(encoder, metal_sel("setTexture:atIndex:"), metal_required(((MetalTexture*)texture.native).texture), (u64)index)
            if program.bindings[index] == (u8)BindingKind.sampled_image:
                # Integer textures are never filtered.
                let filter = Filter.nearest if texture.format == Format.r32_uint else recorded.filters[index]
                metal_bind_texture(encoder, metal_sel("setSamplerState:atIndex:"), metal_sampler_for(drawing, filter), (u64)index)
    var uniforms = recorded.uniforms
    metal_inline_bytes(encoder, metal_sel("setBytes:length:atIndex:"), &uniforms, (u64)uniform_limit, metal_uniform_index)

## A fill: a blit when the value's four bytes are equal (Metal fills bytes),
## otherwise the built-in word kernel, in runs of at most 2^30 words.
func metal_encode_fill(device: MetalDevice*, encoding: MetalEncoding*, recorded: const ComputeCommand*) -> !:
    let target = recorded.buffer else trap("a fill lost its buffer")
    let value = recorded.value
    let low = (u8)(value & 0xff)
    if value == (u32)low * 0x01010101u32:
        metal_fill_buffer(try encoding.transfer(), metal_sel("fillBuffer:range:value:"), target.native, LuceGpuMetalRange((u64)recorded.offset, (u64)recorded.bytes), (u64)low)
        return
    let kernel = try metal_fill_kernel(device)
    let encoder = try encoding.compute()
    metal_object(encoder, metal_sel("setComputePipelineState:"), metal_required(kernel.pipeline))
    let run: usize = 1 << 30
    var done: usize = 0
    let words = recorded.bytes // 4
    while done < words:
        let count = run if words - done > run else words - done
        metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), target.native, (u64)(recorded.offset + done * 4), 0)
        var uniforms: u32[3] = [0, (u32)count, value]
        metal_inline_bytes(encoder, metal_sel("setBytes:length:atIndex:"), &uniforms, 12, metal_uniform_index)
        metal_dispatch(encoder, metal_sel("dispatchThreadgroups:threadsPerThreadgroup:"), LuceGpuMetalExtent((u64)((count + 63) // 64), 1, 1), metal_group(kernel))
        done += count

extern func metal_use_resources as "objc_msgSend"(object: void*, selector: void*, resources: void**, count: u64, usage: u64)
extern func metal_double as "objc_msgSend"(object: void*, selector: void*) -> f64
extern func metal_uint_with as "objc_msgSend"(object: void*, selector: void*, value: u64) -> u64
extern func metal_dispatch as "objc_msgSend"(object: void*, selector: void*, grid: LuceGpuMetalExtent, size: LuceGpuMetalExtent)
extern func metal_dispatch_indirect as "objc_msgSend"(object: void*, selector: void*, arguments: void*, offset: u64, size: LuceGpuMetalExtent)

# mark: Texture tables ========================================================================

## A texture table: a shared buffer of texture handles (MTLResourceID, 8 bytes
## each), which kernels read as an argument buffer at index 29, and the
## textures in it, made resident for each dispatch that reads it.
struct MetalTable:
    var buffer: void*?
    var handles: u64*
    var textures: void*?[]

## The Metal buffer index of a kernel's texture table (embed_shaders.py).
let metal_table_index: u64 = 29

func metal_table_create(device: MetalDevice*, capacity: u32) -> MetalTable*!:
    let pool = objc_autoreleasePoolPush()
    defer objc_autoreleasePoolPop(pool)
    let buffer = metal_new_buffer(metal_required(device.device), metal_sel("newBufferWithLength:options:"), (u64)capacity * 8, metal_storage_shared) else error(out_of_memory, "could not allocate a texture table")
    errdefer metal_void(buffer, metal_sel("release"))
    let contents = metal_ptr(buffer, metal_sel("contents")) else error(failed, "Metal shared buffer storage is inaccessible")
    memory.set(u8[]((u8*)contents, (usize)capacity * 8), 0)
    let textures = try new void*?[(usize)capacity] in memory.heap
    return try new MetalTable(buffer = buffer, handles = (u64*)contents, textures = textures) in memory.heap

func metal_table_set(table: MetalTable*, index: u32, texture: MetalTexture*):
    let native = metal_required(texture.texture)
    table.handles[(usize)index] = metal_uint(native, metal_sel("gpuResourceID"))
    table.textures[(usize)index] = native

func metal_table_clear(table: MetalTable*, index: u32):
    table.handles[(usize)index] = 0
    table.textures[(usize)index] = none

func metal_table_destroy(table: MetalTable*):
    if let buffer = table.buffer:
        metal_void(buffer, metal_sel("release"))
    free(table.textures) in memory.heap
    free(table) in memory.heap

## Bind a table and make its textures resident for the dispatch.
func metal_table_encode(encoder: void*, table: MetalTable*) -> !:
    metal_bind_buffer(encoder, metal_sel("setBuffer:offset:atIndex:"), metal_required(table.buffer), 0, metal_table_index)
    let resident = try new void*[table.textures.length] --- in memory.heap
    defer free(resident) in memory.heap
    var count: usize = 0
    for slot in table.textures:
        if let texture = slot:
            resident[count] = texture
            count += 1
    if count > 0:
        # MTLResourceUsageRead.
        metal_use_resources(encoder, metal_sel("useResources:count:usage:"), resident.data, (u64)count, 1)