Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.2

0.5.2 / src / gpu / vulkan / compute.lucb

#==============================================================================================
#
#   compute - Vulkan compute pipelines, passes and compute buffers
#
#   DESCRIPTION:
#       A kernel is a compute pipeline with its own descriptor set layout (set 0, the kinds its
#       bindings declare) and a 128-byte push-constant range. A pass is one submission on the
#       device's ring: a barrier first makes every earlier submission's writes visible, a
#       barrier between commands makes each see what the ones before it wrote, and a last one
#       hands the results to whatever is submitted next. Its descriptor sets come from one pool
#       retired with the submission; kernels and buffers record it as their last use and are
#       freed only after it completes.
#
#==============================================================================================

struct VulkanKernel:
    var table: bool
    var pipeline: u64
    var pipeline_layout: u64
    var set_layout: u64
    var bindings: u8[16]
    ## The newest submission that dispatches it.
    var last_use: u64

# mark: Limits and buffers ====================================================================

## The device's limits, from VkPhysicalDeviceLimits (and the largest single
## allocation on Vulkan 1.1 and later). Buffers are bound whole, so the largest
## buffer is the largest storage range, in whole words.
func vulkan_device_limits(device: VulkanDevice*) -> Limits:
    var properties = VkPhysicalDeviceProperties2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2)
    var indexing = VkPhysicalDeviceDescriptorIndexingProperties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES)
    var subgroups = VkPhysicalDeviceSubgroupProperties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES)
    if device.version >= vulkan_api_version_1_2:
        subgroups.pNext = (void*)&indexing
    var maintenance = VkPhysicalDeviceMaintenance3Properties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES, pNext = (void*)&subgroups)
    var largest: u64 = 0xffffffffffffffffu64
    if device.version >= vulkan_api_version_1_1:
        properties.pNext = (void*)&maintenance
        vkGetPhysicalDeviceProperties2(device.physical, &properties)
        largest = maintenance.maxMemoryAllocationSize
    else:
        vkGetPhysicalDeviceProperties(device.physical, (void*)&properties.properties)
    let limits = properties.properties.limits
    device.timestamp_period = limits.timestampPeriod
    var buffer_bytes = (u64)limits.maxStorageBufferRange // 4 * 4
    if largest < buffer_bytes: buffer_bytes = largest // 4 * 4
    # Basic, vote, arithmetic, ballot and shuffle, in the compute stage.
    let operations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_VOTE_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT | VK_SUBGROUP_FEATURE_SHUFFLE_BIT
    let subgroup_ops = (subgroups.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 and (subgroups.supportedOperations & operations) == operations
    # A table's descriptors are updated after binding, in one set used by the compute stage.
    var table_size = texture_table_limit
    if indexing.maxDescriptorSetUpdateAfterBindSampledImages < table_size: table_size = indexing.maxDescriptorSetUpdateAfterBindSampledImages
    if indexing.maxPerStageDescriptorUpdateAfterBindSampledImages < table_size: table_size = indexing.maxPerStageDescriptorUpdateAfterBindSampledImages
    let tables = device.features.texture_tables and table_size > 0
    return Limits(buffer_bytes = buffer_bytes, group_threads = limits.maxComputeWorkGroupInvocations, group_size = limits.maxComputeWorkGroupSize,
                  group_count = limits.maxComputeWorkGroupCount, shared_bytes = limits.maxComputeSharedMemorySize, float_atomic_add = device.features.float_atomic_add,
                  subgroup_size = subgroups.subgroupSize, subgroup_ops = subgroup_ops, timestamps = device.timestamp_bits > 0,
                  buffer_addresses = device.features.buffer_addresses, texture_tables = tables, texture_table_size = table_size if tables else 0,
                  ray_query = device.features.ray_query)

func vulkan_limits(device: VulkanDevice*) -> Limits:
    return device.limits

## A zeroed buffer. Device memory is cleared by a fill on the queue; shared
## memory is host-visible and coherent (device-local too where the device has
## such memory and it has room), mapped for good and cleared by the CPU.
func vulkan_buffer_allocate(device: VulkanDevice*, bytes: usize, memory_kind: Memory) -> VulkanBuffer*!:
    try vulkan_drawing(device)
    let state = try new VulkanBuffer(bytes = (u64)bytes) in memory.heap
    errdefer vulkan_buffer_release((void*)device, (void*)state)
    var info = VkBufferCreateInfo(sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, size = state.bytes, usage = vulkan_buffer_usage(device))
    try vulkan_check(vkCreateBuffer(device.device, &info, none, &state.buffer), "a Vulkan buffer could not be created")
    var requirements: VkMemoryRequirements
    vkGetBufferMemoryRequirements(device.device, state.buffer, &requirements)
    if memory_kind == Memory.device:
        state.memory = try vulkan_buffer_memory(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
        try vulkan_check(vkBindBufferMemory(device.device, state.buffer, state.memory, 0), "Vulkan buffer memory could not be bound")
        let slot = try vulkan_begin(device)
        vkCmdFillBuffer(slot.command, state.buffer, 0, state.bytes, 0)
        vulkan_compute_barrier(slot.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)
        state.last_use = try vulkan_end(device, slot)
        return state
    let visible = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
    let local = visible | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
    var block: u64 = 0
    if vulkan_has_memory_type(device, requirements, local):
        block = vulkan_buffer_memory(device, requirements, local) catch failure:
            _ = failure
            recover 0
    if block == 0:
        block = try vulkan_buffer_memory(device, requirements, visible)
    state.memory = block
    try vulkan_check(vkBindBufferMemory(device.device, state.buffer, state.memory, 0), "Vulkan buffer memory could not be bound")
    try vulkan_check(vkMapMemory(device.device, state.memory, 0, state.bytes, 0, &state.mapped), "Vulkan buffer memory could not be mapped")
    let mapped = state.mapped else error(failed, "Vulkan returned an empty memory mapping")
    memory.set(u8[]((u8*)mapped, bytes), 0)
    return state

## Dedicated memory for a buffer, allocated for device addresses where the
## device has them.
func vulkan_buffer_memory(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u64!:
    let type_index = try vulkan_memory_type(device, requirements, flags)
    var addressed = VkMemoryAllocateFlagsInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT)
    var info = VkMemoryAllocateInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, allocationSize = requirements.size, memoryTypeIndex = type_index)
    if device.features.buffer_addresses:
        info.pNext = (void*)&addressed
    var block: u64 = 0
    try vulkan_check(vkAllocateMemory(device.device, &info, none, &block), "Vulkan device memory allocation failed")
    return block

func vulkan_buffer_address(device: VulkanDevice*, buffer: VulkanBuffer*) -> u64:
    var info = VkBufferDeviceAddressInfo(sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, buffer = buffer.buffer)
    return vkGetBufferDeviceAddress(device.device, &info)

## A shared buffer's mapping.
func vulkan_buffer_contents(buffer: VulkanBuffer*) -> u8*?:
    let mapped = buffer.mapped else return none
    return (u8*)mapped

# mark: Kernels ===============================================================================

## The device state kernels share, made on the main thread before a kernel
## builds on another (vulkan_kernel_create then touches none of it).
func vulkan_kernel_prepare(device: VulkanDevice*, table: bool) -> !:
    try vulkan_drawing(device)
    if table: _ = try vulkan_table_layout(device)

func vulkan_kernel_create(device: VulkanDevice*, spirv: const u32[], bindings: u8[16], table: bool, constants: const Constant[]) -> void*!:
    try vulkan_kernel_prepare(device, table)
    let state = try new VulkanKernel(bindings = bindings, table = table) in memory.heap
    errdefer vulkan_kernel_destroy(device, state)
    var layout_bindings: VkDescriptorSetLayoutBinding[16]
    var count: u32 = 0
    for index in 0..<binding_limit:
        if bindings[index] == 0:
            continue
        layout_bindings[(usize)count] = VkDescriptorSetLayoutBinding(binding = (u32)index, descriptorType = vulkan_descriptor_type(bindings[index]),
                                                                    descriptorCount = 1, stageFlags = VK_SHADER_STAGE_COMPUTE_BIT)
        count += 1
    var set_info = VkDescriptorSetLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, bindingCount = count, pBindings = &layout_bindings[0])
    try vulkan_check(vkCreateDescriptorSetLayout(device.device, &set_info, none, &state.set_layout), "a Vulkan kernel layout could not be created")
    var push = VkPushConstantRange(stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, size = (u32)uniform_limit)
    # A kernel reading a texture table has the device's table layout as set 1.
    var set_layouts: u64[2] = [state.set_layout, try vulkan_table_layout(device) if table else 0]
    var layout_info = VkPipelineLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, setLayoutCount = 2 if table else 1, pSetLayouts = &set_layouts[0],
                                                 pushConstantRangeCount = 1, pPushConstantRanges = &push)
    try vulkan_check(vkCreatePipelineLayout(device.device, &layout_info, none, &state.pipeline_layout), "a Vulkan kernel pipeline layout could not be created")
    let module = vulkan_shader(device, spirv) catch failure:
        error(invalid_shader, "Vulkan refused the kernel's SPIR-V")
    defer vkDestroyShaderModule(device.device, module, none)
    # Specialization constants: four bytes each (VkBool32 for bool), in order.
    var entries: VkSpecializationMapEntry[64]
    var data: u32[64]
    if constants.length > entries.length:
        error(invalid_shader, "a kernel is given at most 64 specialization constants")
    for index in 0..<constants.length:
        entries[index] = VkSpecializationMapEntry(constantID = constants[index].id, offset = (u32)index * 4, size = 4)
        data[index] = constants[index].bits
    var specialization = VkSpecializationInfo(mapEntryCount = (u32)constants.length, pMapEntries = &entries[0], dataSize = constants.length * 4, pData = (void*)&data[0])
    var pipeline_info = VkComputePipelineCreateInfo(sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, layout = state.pipeline_layout,
                                                    stage = VkPipelineShaderStageCreateInfo(sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
                                                                                            stage = VK_SHADER_STAGE_COMPUTE_BIT, module = module, pName = "main"))
    if constants.length > 0:
        pipeline_info.stage.pSpecializationInfo = &specialization
    if vkCreateComputePipelines(device.device, 0, 1, &pipeline_info, none, &state.pipeline) != VK_SUCCESS:
        error(invalid_shader, "Vulkan could not build the kernel's compute pipeline")
    return (void*)state

func vulkan_descriptor_type(kind: u8) -> u32:
    if kind == (u8)BindingKind.storage_image: return VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
    if kind == (u8)BindingKind.sampled_image: return VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER
    if kind == (u8)BindingKind.sampler: return VK_DESCRIPTOR_TYPE_SAMPLER
    if kind == (u8)BindingKind.acceleration_structure: return VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR
    return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER

## Release once every submission that dispatched the kernel has completed.
func vulkan_kernel_release(raw_device: void*, raw_state: void*):
    let device = (VulkanDevice*)raw_device
    let state = (VulkanKernel*)raw_state
    let item = VulkanRetired(kind = vulkan_retired_kernel, handle = state.pipeline, view = state.pipeline_layout, extra = state.set_layout)
    let serial = state.last_use
    free(state) in memory.heap
    vulkan_poll(device)
    if serial <= device.completed:
        vulkan_retired_free(device, item)
        return
    vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
        vulkan_wait(device, serial) catch waited: _ = waited
        vulkan_retired_free(device, item)

func vulkan_kernel_destroy(device: VulkanDevice*, state: VulkanKernel*):
    vulkan_retired_free(device, VulkanRetired(kind = vulkan_retired_kernel, handle = state.pipeline, view = state.pipeline_layout, extra = state.set_layout))
    free(state) in memory.heap

# mark: Passes ================================================================================

## Every write by `source` stages visible to every read and write by `target`
## stages: what orders a pass's commands, and a pass against its neighbours.
func vulkan_compute_barrier(command: u64, source: u32, target: u32):
    var barrier = VkMemoryBarrier(sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
                                  dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT)
    vkCmdPipelineBarrier(command, source, target, 0, 1, (void*)&barrier, 0, none, 0, none)

## The stages a pass's commands run in: acceleration structure builds too,
## where ray queries are enabled (the stage is invalid otherwise).
func vulkan_compute_stages(device: VulkanDevice*) -> u32:
    let stages = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT
    return stages | VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR if device.features.ray_query else stages

## Record `commands` into one submission and submit it without waiting. Like
## every submission its start and end are timed (vulkan_begin, vulkan_end); a
## pass with timestamps gets a query pool of its own, kept for reading them.
func vulkan_compute_submit(device: VulkanDevice*, commands: const ComputeCommand[], stamps: usize, addressed: BufferState*[]) -> ComputeSubmitted!:
    try vulkan_drawing(device)
    var dispatches: u32 = 0
    for recorded in commands:
        if recorded.kind == command_dispatch or recorded.kind == command_indirect: dispatches += 1
    var stamped: u64 = 0
    if stamps > 0:
        stamped = try vulkan_query_pool(device, (u32)stamps)
    errdefer vulkan_query_pool_drop(device, stamped)
    let slot = try vulkan_begin(device)
    var pool: u64 = 0
    errdefer vulkan_compute_abandon(device, slot, pool)
    if stamped != 0:
        vkCmdResetQueryPool(slot.command, stamped, 0, (u32)stamps)
    if dispatches > 0:
        var sizes: VkDescriptorPoolSize[5] = [
            VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = dispatches * (u32)binding_limit),
            VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, descriptorCount = dispatches * (u32)binding_limit),
            VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, descriptorCount = dispatches * (u32)binding_limit),
            VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER, descriptorCount = dispatches * (u32)binding_limit),
            VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR, descriptorCount = dispatches * (u32)binding_limit)]
        # Acceleration structure descriptors exist only where ray queries are enabled.
        let kinds: u32 = 5 if device.features.ray_query else 4
        var pool_info = VkDescriptorPoolCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, maxSets = dispatches, poolSizeCount = kinds, pPoolSizes = &sizes[0])
        try vulkan_check(vkCreateDescriptorPool(device.device, &pool_info, none, &pool), "Vulkan compute descriptors could not be created")
    # One scratch buffer for the pass's builds, ordered by the barriers between them.
    var scratch: u64 = 0
    let scratch_bytes = vulkan_scratch_bytes(commands)
    if scratch_bytes > 0:
        let alignment = vulkan_ray(device).scratch_alignment
        let memory_block = try vulkan_transient(device, slot, scratch_bytes + alignment, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, false)
        scratch = (memory_block.address + alignment - 1) // alignment * alignment
    # Earlier submissions' writes (uploads, passes) before this pass reads.
    vulkan_compute_barrier(slot.command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, vulkan_compute_stages(device))
    for index in 0..<commands.length:
        let recorded = &commands[index]
        if index > 0:
            vulkan_compute_barrier(slot.command, vulkan_compute_stages(device), vulkan_compute_stages(device))
        if recorded.kind == command_timestamp:
            # After every earlier command has finished.
            vkCmdWriteTimestamp(slot.command, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, stamped, recorded.value)
        elif recorded.kind == command_copy:
            let origin = recorded.buffer else trap("a copy lost its source")
            let goal = recorded.target else trap("a copy lost its destination")
            var region = VkBufferCopy(srcOffset = (u64)recorded.offset, dstOffset = (u64)recorded.target_offset, size = (u64)recorded.bytes)
            vkCmdCopyBuffer(slot.command, ((VulkanBuffer*)origin.native).buffer, ((VulkanBuffer*)goal.native).buffer, 1, &region)
        elif recorded.kind == command_fill:
            let target = recorded.buffer else trap("a fill lost its buffer")
            vkCmdFillBuffer(slot.command, ((VulkanBuffer*)target.native).buffer, (u64)recorded.offset, (u64)recorded.bytes, recorded.value)
        elif recorded.kind >= command_build_blas:
            try vulkan_encode_build(device, slot, slot.command, recorded, scratch)
        else:
            try vulkan_encode_dispatch(device, slot.command, pool, recorded)
    # Storage images back to sampling, the layout the rest of the device expects.
    for recorded in commands:
        for bound in recorded.textures:
            if let texture = bound: vulkan_transition(slot.command, (VulkanTexture*)texture.native, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    # The results, to whatever is submitted next: frames, reads, other passes.
    vulkan_compute_barrier(slot.command, vulkan_compute_stages(device), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)
    if pool != 0:
        try vulkan_retire(slot, VulkanRetired(kind = vulkan_retired_descriptors, handle = pool))
        pool = 0
    let serial = try vulkan_end(device, slot)
    for recorded in commands:
        if let kernel = recorded.kernel: ((VulkanKernel*)kernel.built()).last_use = serial
        for bound in recorded.buffers:
            if let buffer = bound: ((VulkanBuffer*)buffer.native).last_use = serial
        for bound in recorded.textures:
            if let texture = bound: ((VulkanTexture*)texture.native).last_use = serial
        if let table = recorded.table: vulkan_table_used((VulkanTable*)table.native, serial)
        if let accel = recorded.accel: vulkan_accel_used(accel, serial)
        for bound in recorded.accels:
            if let accel = bound: vulkan_accel_used(accel, serial)
        for blas in recorded.instanced:
            ((VulkanAccel*)blas.native).last_use = serial
        if let buffer = recorded.buffer: ((VulkanBuffer*)buffer.native).last_use = serial
        if let buffer = recorded.target: ((VulkanBuffer*)buffer.native).last_use = serial
    # Buffers kernels may reach by address count as used by every pass.
    for buffer in addressed:
        ((VulkanBuffer*)buffer.native).last_use = serial
    # A pass is submitted when the program submits it, so the GPU starts now.
    try vulkan_flush(device)
    return ComputeSubmitted(serial = serial, native = (void*)(usize)stamped if stamped != 0 else none)

# mark: Timing ================================================================================

func vulkan_query_pool_drop(device: VulkanDevice*, pool: u64):
    if pool != 0: vkDestroyQueryPool(device.device, pool, none)

func vulkan_query_pool(device: VulkanDevice*, count: u32) -> u64!:
    var info = VkQueryPoolCreateInfo(sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, queryType = VK_QUERY_TYPE_TIMESTAMP, queryCount = count)
    var pool: u64 = 0
    try vulkan_check(vkCreateQueryPool(device.device, &info, none, &pool), "a Vulkan timestamp query pool could not be created")
    return pool

## Finished queries in nanoseconds: the queue's valid bits, scaled by the
## device's tick period.
func vulkan_read_queries(device: VulkanDevice*, pool: u64, first: u32, out: u64[]) -> bool:
    let bytes = out.length * 8
    if vkGetQueryPoolResults(device.device, pool, first, (u32)out.length, bytes, (void*)out.data, 8, VK_QUERY_RESULT_64_BIT) != VK_SUCCESS:
        return false
    let mask = 0xffffffffffffffffu64 if device.timestamp_bits >= 64 else (1u64 << (u64)device.timestamp_bits) - 1
    for index in 0..<out.length:
        out[index] = (u64)((f64)(out[index] & mask) * (f64)device.timestamp_period)
    return true

## The start and end of a finished submission timed in the device's queries,
## while its ring slot still holds them.
func vulkan_submission_time(device: VulkanDevice*, serial: u64) -> GpuTime?:
    let index = (usize)(serial % (u64)vulkan_slot_limit)
    if serial == 0 or device.timing == 0 or device.timing_serial[index] != serial:
        return none
    var times: u64[2]
    if not vulkan_read_queries(device, device.timing, (u32)index * 2, times):
        return none
    return GpuTime(start = times[0], end = times[1])

func vulkan_compute_stamps(device: VulkanDevice*, native: void*?, out: u64[]) -> !:
    let pool = (u64)(usize)(native else error(invalid_geometry, "the pass recorded no timestamps"))
    if not vulkan_read_queries(device, pool, 0, out):
        error(failed, "Vulkan timestamps could not be read")

## Free a pass's query pool once its submission has finished.
func vulkan_compute_release(device: VulkanDevice*, native: void*, serial: u64):
    let item = VulkanRetired(kind = vulkan_retired_queries, handle = (u64)(usize)native)
    vulkan_poll(device)
    if serial <= device.completed:
        vulkan_retired_free(device, item)
        return
    vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
        vulkan_wait(device, serial) catch waited: _ = waited
        vulkan_retired_free(device, item)

## After a failure before submission: end the recording unsubmitted and free the pool.
func vulkan_compute_abandon(device: VulkanDevice*, slot: VulkanSlot*, pool: u64):
    # The slot's recording is reset when it is next begun.
    _ = slot
    if pool != 0: vkDestroyDescriptorPool(device.device, pool, none)

## Bind a dispatch's pipeline, a descriptor set of its bindings and its
## uniforms, and dispatch it directly or from its arguments buffer.
func vulkan_encode_dispatch(device: VulkanDevice*, command: u64, pool: u64, recorded: const ComputeCommand*) -> !:
    let program = recorded.kernel else trap("a dispatch lost its kernel")
    let kernel = (VulkanKernel*)program.built()
    var set: u64 = 0
    var set_info = VkDescriptorSetAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, descriptorPool = pool, descriptorSetCount = 1, pSetLayouts = &kernel.set_layout)
    try vulkan_check(vkAllocateDescriptorSets(device.device, &set_info, &set), "Vulkan compute descriptors could not be allocated")
    var infos: VkDescriptorBufferInfo[16]
    var images: VkDescriptorImageInfo[16]
    var writes: VkWriteDescriptorSet[16]
    var count: u32 = 0
    for index in 0..<binding_limit:
        if program.bindings[index] == (u8)BindingKind.sampler:
            images[index] = VkDescriptorImageInfo(sampler = vulkan_sampler_for(device, recorded.filters[index], true))
            writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
                                                        descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER, pImageInfo = &images[index])
            count += 1
            continue
        if let buffer = recorded.buffers[index]:
            let native = (VulkanBuffer*)buffer.native
            infos[index] = VkDescriptorBufferInfo(buffer = native.buffer, range = native.bytes)
            writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
                                                        descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, pBufferInfo = &infos[index])
            count += 1
            continue
        if let accel = recorded.accels[index]:
            vulkan_accel_write(device, set, (u32)index, accel)
            continue
        let texture = recorded.textures[index] else continue
        let image = (VulkanTexture*)texture.native
        # Storage images are loaded and stored in the general layout; sampled
        # ones are read as every draw reads them.
        let storage = program.bindings[index] == (u8)BindingKind.storage_image
        let layout = VK_IMAGE_LAYOUT_GENERAL if storage else VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
        vulkan_transition(command, image, layout)
        let sampler = vulkan_sampler_for(device, recorded.filters[index], image.filterable and texture.format != Format.r32_uint)
        images[index] = VkDescriptorImageInfo(sampler = 0 if storage else sampler, imageView = image.view, imageLayout = layout)
        writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
                                                    descriptorType = vulkan_descriptor_type(program.bindings[index]), pImageInfo = &images[index])
        count += 1
    if count > 0:
        vkUpdateDescriptorSets(device.device, count, &writes[0], 0, none)
    vkCmdBindPipeline(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline)
    vkCmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline_layout, 0, 1, &set, 0, none)
    if let table = recorded.table:
        let native = (VulkanTable*)table.native
        vulkan_table_ready(command, native)
        vkCmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline_layout, 1, 1, &native.set, 0, none)
    var uniforms = recorded.uniforms
    vkCmdPushConstants(command, kernel.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, (u32)uniform_limit, (void*)&uniforms)
    if recorded.kind == command_indirect:
        let arguments = recorded.buffer else trap("an indirect dispatch lost its arguments")
        vkCmdDispatchIndirect(command, ((VulkanBuffer*)arguments.native).buffer, (u64)recorded.offset)
    else:
        vkCmdDispatch(command, recorded.groups[0], recorded.groups[1], recorded.groups[2])

# mark: Texture tables ========================================================================

## A texture table: one descriptor set of sampled images, updated after
## binding and partially bound, from a pool of its own; the textures in it,
## made ready for sampling by each dispatch that reads it.
struct VulkanTable:
    var pool: u64
    var set: u64
    var textures: VulkanTexture*?[]
    ## The newest submission that read the table.
    var last_use: u64

## The device's table layout, set 1 of every kernel that reads a table: a
## variable-count array of sampled images, made on first use.
func vulkan_table_layout(device: VulkanDevice*) -> u64!:
    if device.table_layout != 0:
        return device.table_layout
    var binding = VkDescriptorSetLayoutBinding(binding = 0, descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, descriptorCount = device.limits.texture_table_size,
                                               stageFlags = VK_SHADER_STAGE_COMPUTE_BIT)
    var flags: u32 = VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT | VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT | VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT
    var binding_flags = VkDescriptorSetLayoutBindingFlagsCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, bindingCount = 1, pBindingFlags = &flags)
    var info = VkDescriptorSetLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, pNext = (void*)&binding_flags,
                                               flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT, bindingCount = 1, pBindings = &binding)
    try vulkan_check(vkCreateDescriptorSetLayout(device.device, &info, none, &device.table_layout), "the Vulkan texture table layout could not be created")
    return device.table_layout

func vulkan_table_create(device: VulkanDevice*, capacity: u32) -> VulkanTable*!:
    try vulkan_drawing(device)
    var layout = try vulkan_table_layout(device)
    let textures = try new VulkanTexture*?[(usize)capacity] in memory.heap
    let state = try new VulkanTable(textures = textures) in memory.heap
    errdefer vulkan_table_destroy(device, state)
    var size = VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, descriptorCount = capacity)
    var pool_info = VkDescriptorPoolCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, flags = VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT,
                                               maxSets = 1, poolSizeCount = 1, pPoolSizes = &size)
    try vulkan_check(vkCreateDescriptorPool(device.device, &pool_info, none, &state.pool), "a Vulkan texture table could not be created")
    var count = capacity
    var variable = VkDescriptorSetVariableDescriptorCountAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO, descriptorSetCount = 1, pDescriptorCounts = &count)
    var set_info = VkDescriptorSetAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, pNext = (void*)&variable, descriptorPool = state.pool,
                                               descriptorSetCount = 1, pSetLayouts = &layout)
    try vulkan_check(vkAllocateDescriptorSets(device.device, &set_info, &state.set), "a Vulkan texture table could not be allocated")
    return state

## Point slot `index` at `image` (or, cleared, at the placeholder image, so no
## slot ever names a destroyed view).
func vulkan_table_write(device: VulkanDevice*, table: VulkanTable*, index: u32, image: VulkanTexture*):
    var info = VkDescriptorImageInfo(imageView = image.view, imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    var write = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = table.set, dstBinding = 0, dstArrayElement = index, descriptorCount = 1,
                                     descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, pImageInfo = &info)
    vkUpdateDescriptorSets(device.device, 1, &write, 0, none)

func vulkan_table_set(device: VulkanDevice*, table: VulkanTable*, index: u32, texture: VulkanTexture*):
    vulkan_table_write(device, table, index, texture)
    table.textures[(usize)index] = texture

func vulkan_table_clear(device: VulkanDevice*, table: VulkanTable*, index: u32):
    let placeholder = device.placeholder else trap("Vulkan drawing state has no placeholder image")
    vulkan_table_write(device, table, index, placeholder)
    table.textures[(usize)index] = none

## Every texture in the table in the sampling layout, before a dispatch reads it.
func vulkan_table_ready(command: u64, table: VulkanTable*):
    for slot in table.textures:
        if let texture = slot: vulkan_transition(command, texture, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)

func vulkan_table_used(table: VulkanTable*, serial: u64):
    table.last_use = serial
    for slot in table.textures:
        if let texture = slot: texture.last_use = serial

## Free the table's pool (and its set) once its last submission has finished.
func vulkan_table_destroy(device: VulkanDevice*, table: VulkanTable*):
    let item = VulkanRetired(kind = vulkan_retired_descriptors, handle = table.pool)
    let serial = table.last_use
    free(table.textures) in memory.heap
    free(table) in memory.heap
    if item.handle == 0: return
    vulkan_poll(device)
    if serial <= device.completed:
        vulkan_retired_free(device, item)
        return
    vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
        vulkan_wait(device, serial) catch waited: _ = waited
        vulkan_retired_free(device, item)