dymokomi/luce-gpu
Source of release 0.5.3
0.5.3 / src / gpu / vulkan / compute.lucb
#==============================================================================================
#
# compute - Vulkan compute pipelines, passes and compute buffers
#
# DESCRIPTION:
# A kernel is a compute pipeline with its own descriptor set layout (set 0, the kinds its
# bindings declare) and a 128-byte push-constant range. A pass is one submission on the
# device's ring: a barrier first makes every earlier submission's writes visible, a
# barrier between commands makes each see what the ones before it wrote, and a last one
# hands the results to whatever is submitted next. Its descriptor sets come from one pool
# retired with the submission; kernels and buffers record it as their last use and are
# freed only after it completes.
#
#==============================================================================================
struct VulkanKernel:
var table: bool
var pipeline: u64
var pipeline_layout: u64
var set_layout: u64
var bindings: u8[16]
## The newest submission that dispatches it.
var last_use: u64
# mark: Limits and buffers ====================================================================
## The device's limits, from VkPhysicalDeviceLimits (and the largest single
## allocation on Vulkan 1.1 and later). Buffers are bound whole, so the largest
## buffer is the largest storage range, in whole words.
func vulkan_device_limits(device: VulkanDevice*) -> Limits:
var properties = VkPhysicalDeviceProperties2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2)
var indexing = VkPhysicalDeviceDescriptorIndexingProperties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES)
var subgroups = VkPhysicalDeviceSubgroupProperties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES)
if device.version >= vulkan_api_version_1_2:
subgroups.pNext = (void*)&indexing
var maintenance = VkPhysicalDeviceMaintenance3Properties(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES, pNext = (void*)&subgroups)
var largest: u64 = 0xffffffffffffffffu64
if device.version >= vulkan_api_version_1_1:
properties.pNext = (void*)&maintenance
vkGetPhysicalDeviceProperties2(device.physical, &properties)
largest = maintenance.maxMemoryAllocationSize
else:
vkGetPhysicalDeviceProperties(device.physical, (void*)&properties.properties)
let limits = properties.properties.limits
device.timestamp_period = limits.timestampPeriod
var buffer_bytes = (u64)limits.maxStorageBufferRange // 4 * 4
if largest < buffer_bytes: buffer_bytes = largest // 4 * 4
# Basic, vote, arithmetic, ballot and shuffle, in the compute stage.
let operations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_VOTE_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT | VK_SUBGROUP_FEATURE_SHUFFLE_BIT
let subgroup_ops = (subgroups.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 and (subgroups.supportedOperations & operations) == operations
# A table's descriptors are updated after binding, in one set used by the compute stage.
var table_size = texture_table_limit
if indexing.maxDescriptorSetUpdateAfterBindSampledImages < table_size: table_size = indexing.maxDescriptorSetUpdateAfterBindSampledImages
if indexing.maxPerStageDescriptorUpdateAfterBindSampledImages < table_size: table_size = indexing.maxPerStageDescriptorUpdateAfterBindSampledImages
let tables = device.features.texture_tables and table_size > 0
return Limits(buffer_bytes = buffer_bytes, group_threads = limits.maxComputeWorkGroupInvocations, group_size = limits.maxComputeWorkGroupSize,
group_count = limits.maxComputeWorkGroupCount, shared_bytes = limits.maxComputeSharedMemorySize, float_atomic_add = device.features.float_atomic_add,
subgroup_size = subgroups.subgroupSize, subgroup_ops = subgroup_ops, timestamps = device.timestamp_bits > 0,
buffer_addresses = device.features.buffer_addresses, texture_tables = tables, texture_table_size = table_size if tables else 0,
ray_query = device.features.ray_query)
func vulkan_limits(device: VulkanDevice*) -> Limits:
return device.limits
## A zeroed buffer. Device memory is cleared by a fill on the queue; shared
## memory is host-visible and coherent (device-local too where the device has
## such memory and it has room), mapped for good and cleared by the CPU.
func vulkan_buffer_allocate(device: VulkanDevice*, bytes: usize, memory_kind: Memory) -> VulkanBuffer*!:
try vulkan_drawing(device)
let state = try new VulkanBuffer(bytes = (u64)bytes) in memory.heap
errdefer vulkan_buffer_release((void*)device, (void*)state)
var info = VkBufferCreateInfo(sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, size = state.bytes, usage = vulkan_buffer_usage(device))
try vulkan_check(vkCreateBuffer(device.device, &info, none, &state.buffer), "a Vulkan buffer could not be created")
var requirements: VkMemoryRequirements
vkGetBufferMemoryRequirements(device.device, state.buffer, &requirements)
if memory_kind == Memory.device:
state.memory = try vulkan_buffer_memory(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
try vulkan_check(vkBindBufferMemory(device.device, state.buffer, state.memory, 0), "Vulkan buffer memory could not be bound")
let slot = try vulkan_begin(device)
vkCmdFillBuffer(slot.command, state.buffer, 0, state.bytes, 0)
vulkan_compute_barrier(slot.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)
state.last_use = try vulkan_end(device, slot)
return state
let visible = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
let local = visible | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
var block: u64 = 0
if vulkan_has_memory_type(device, requirements, local):
block = vulkan_buffer_memory(device, requirements, local) catch failure:
_ = failure
recover 0
if block == 0:
block = try vulkan_buffer_memory(device, requirements, visible)
state.memory = block
try vulkan_check(vkBindBufferMemory(device.device, state.buffer, state.memory, 0), "Vulkan buffer memory could not be bound")
try vulkan_check(vkMapMemory(device.device, state.memory, 0, state.bytes, 0, &state.mapped), "Vulkan buffer memory could not be mapped")
let mapped = state.mapped else error(failed, "Vulkan returned an empty memory mapping")
memory.set(u8[]((u8*)mapped, bytes), 0)
return state
## Dedicated memory for a buffer, allocated for device addresses where the
## device has them.
func vulkan_buffer_memory(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u64!:
let type_index = try vulkan_memory_type(device, requirements, flags)
var addressed = VkMemoryAllocateFlagsInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT)
var info = VkMemoryAllocateInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, allocationSize = requirements.size, memoryTypeIndex = type_index)
if device.features.buffer_addresses:
info.pNext = (void*)&addressed
var block: u64 = 0
try vulkan_check(vkAllocateMemory(device.device, &info, none, &block), "Vulkan device memory allocation failed")
return block
func vulkan_buffer_address(device: VulkanDevice*, buffer: VulkanBuffer*) -> u64:
var info = VkBufferDeviceAddressInfo(sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, buffer = buffer.buffer)
return vkGetBufferDeviceAddress(device.device, &info)
## A shared buffer's mapping.
func vulkan_buffer_contents(buffer: VulkanBuffer*) -> u8*?:
let mapped = buffer.mapped else return none
return (u8*)mapped
# mark: Kernels ===============================================================================
## The device state kernels share, made on the main thread before a kernel
## builds on another (vulkan_kernel_create then touches none of it).
func vulkan_kernel_prepare(device: VulkanDevice*, table: bool) -> !:
try vulkan_drawing(device)
if table: _ = try vulkan_table_layout(device)
func vulkan_kernel_create(device: VulkanDevice*, spirv: const u32[], bindings: u8[16], table: bool, constants: const Constant[]) -> void*!:
try vulkan_kernel_prepare(device, table)
let state = try new VulkanKernel(bindings = bindings, table = table) in memory.heap
errdefer vulkan_kernel_destroy(device, state)
var layout_bindings: VkDescriptorSetLayoutBinding[16]
var count: u32 = 0
for index in 0..<binding_limit:
if bindings[index] == 0:
continue
layout_bindings[(usize)count] = VkDescriptorSetLayoutBinding(binding = (u32)index, descriptorType = vulkan_descriptor_type(bindings[index]),
descriptorCount = 1, stageFlags = VK_SHADER_STAGE_COMPUTE_BIT)
count += 1
var set_info = VkDescriptorSetLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, bindingCount = count, pBindings = &layout_bindings[0])
try vulkan_check(vkCreateDescriptorSetLayout(device.device, &set_info, none, &state.set_layout), "a Vulkan kernel layout could not be created")
var push = VkPushConstantRange(stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, size = (u32)uniform_limit)
# A kernel reading a texture table has the device's table layout as set 1.
var set_layouts: u64[2] = [state.set_layout, try vulkan_table_layout(device) if table else 0]
var layout_info = VkPipelineLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, setLayoutCount = 2 if table else 1, pSetLayouts = &set_layouts[0],
pushConstantRangeCount = 1, pPushConstantRanges = &push)
try vulkan_check(vkCreatePipelineLayout(device.device, &layout_info, none, &state.pipeline_layout), "a Vulkan kernel pipeline layout could not be created")
let module = vulkan_shader(device, spirv) catch failure:
error(invalid_shader, "Vulkan refused the kernel's SPIR-V")
defer vkDestroyShaderModule(device.device, module, none)
# Specialization constants: four bytes each (VkBool32 for bool), in order.
var entries: VkSpecializationMapEntry[64]
var data: u32[64]
if constants.length > entries.length:
error(invalid_shader, "a kernel is given at most 64 specialization constants")
for index in 0..<constants.length:
entries[index] = VkSpecializationMapEntry(constantID = constants[index].id, offset = (u32)index * 4, size = 4)
data[index] = constants[index].bits
var specialization = VkSpecializationInfo(mapEntryCount = (u32)constants.length, pMapEntries = &entries[0], dataSize = constants.length * 4, pData = (void*)&data[0])
var pipeline_info = VkComputePipelineCreateInfo(sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, layout = state.pipeline_layout,
stage = VkPipelineShaderStageCreateInfo(sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
stage = VK_SHADER_STAGE_COMPUTE_BIT, module = module, pName = "main"))
if constants.length > 0:
pipeline_info.stage.pSpecializationInfo = &specialization
if vkCreateComputePipelines(device.device, 0, 1, &pipeline_info, none, &state.pipeline) != VK_SUCCESS:
error(invalid_shader, "Vulkan could not build the kernel's compute pipeline")
return (void*)state
func vulkan_descriptor_type(kind: u8) -> u32:
if kind == (u8)BindingKind.storage_image: return VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
if kind == (u8)BindingKind.sampled_image: return VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER
if kind == (u8)BindingKind.sampler: return VK_DESCRIPTOR_TYPE_SAMPLER
if kind == (u8)BindingKind.acceleration_structure: return VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR
return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
## Release once every submission that dispatched the kernel has completed.
func vulkan_kernel_release(raw_device: void*, raw_state: void*):
let device = (VulkanDevice*)raw_device
let state = (VulkanKernel*)raw_state
let item = VulkanRetired(kind = vulkan_retired_kernel, handle = state.pipeline, view = state.pipeline_layout, extra = state.set_layout)
let serial = state.last_use
free(state) in memory.heap
vulkan_poll(device)
if serial <= device.completed:
vulkan_retired_free(device, item)
return
vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
vulkan_wait(device, serial) catch waited: _ = waited
vulkan_retired_free(device, item)
func vulkan_kernel_destroy(device: VulkanDevice*, state: VulkanKernel*):
vulkan_retired_free(device, VulkanRetired(kind = vulkan_retired_kernel, handle = state.pipeline, view = state.pipeline_layout, extra = state.set_layout))
free(state) in memory.heap
# mark: Passes ================================================================================
## Every write by `source` stages visible to every read and write by `target`
## stages: what orders a pass's commands, and a pass against its neighbours.
func vulkan_compute_barrier(command: u64, source: u32, target: u32):
var barrier = VkMemoryBarrier(sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT)
vkCmdPipelineBarrier(command, source, target, 0, 1, (void*)&barrier, 0, none, 0, none)
## The stages a pass's commands run in: acceleration structure builds too,
## where ray queries are enabled (the stage is invalid otherwise).
func vulkan_compute_stages(device: VulkanDevice*) -> u32:
let stages = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT
return stages | VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR if device.features.ray_query else stages
## Record `commands` into one submission and submit it without waiting. Like
## every submission its start and end are timed (vulkan_begin, vulkan_end); a
## pass with timestamps gets a query pool of its own, kept for reading them.
func vulkan_compute_submit(device: VulkanDevice*, commands: const ComputeCommand[], stamps: usize, addressed: BufferState*[]) -> ComputeSubmitted!:
try vulkan_drawing(device)
var dispatches: u32 = 0
for recorded in commands:
if recorded.kind == command_dispatch or recorded.kind == command_indirect: dispatches += 1
var stamped: u64 = 0
if stamps > 0:
stamped = try vulkan_query_pool(device, (u32)stamps)
errdefer vulkan_query_pool_drop(device, stamped)
let slot = try vulkan_begin(device)
var pool: u64 = 0
errdefer vulkan_compute_abandon(device, slot, pool)
if stamped != 0:
vkCmdResetQueryPool(slot.command, stamped, 0, (u32)stamps)
if dispatches > 0:
var sizes: VkDescriptorPoolSize[5] = [
VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = dispatches * (u32)binding_limit),
VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, descriptorCount = dispatches * (u32)binding_limit),
VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, descriptorCount = dispatches * (u32)binding_limit),
VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER, descriptorCount = dispatches * (u32)binding_limit),
VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR, descriptorCount = dispatches * (u32)binding_limit)]
# Acceleration structure descriptors exist only where ray queries are enabled.
let kinds: u32 = 5 if device.features.ray_query else 4
var pool_info = VkDescriptorPoolCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, maxSets = dispatches, poolSizeCount = kinds, pPoolSizes = &sizes[0])
try vulkan_check(vkCreateDescriptorPool(device.device, &pool_info, none, &pool), "Vulkan compute descriptors could not be created")
# One scratch buffer for the pass's builds, ordered by the barriers between them.
var scratch: u64 = 0
let scratch_bytes = vulkan_scratch_bytes(commands)
if scratch_bytes > 0:
let alignment = vulkan_ray(device).scratch_alignment
let memory_block = try vulkan_transient(device, slot, scratch_bytes + alignment, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, false)
scratch = (memory_block.address + alignment - 1) // alignment * alignment
# Earlier submissions' writes (uploads, passes) before this pass reads.
vulkan_compute_barrier(slot.command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, vulkan_compute_stages(device))
for index in 0..<commands.length:
let recorded = &commands[index]
if index > 0:
vulkan_compute_barrier(slot.command, vulkan_compute_stages(device), vulkan_compute_stages(device))
if recorded.kind == command_timestamp:
# After every earlier command has finished.
vkCmdWriteTimestamp(slot.command, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, stamped, recorded.value)
elif recorded.kind == command_copy:
let origin = recorded.buffer else trap("a copy lost its source")
let goal = recorded.target else trap("a copy lost its destination")
var region = VkBufferCopy(srcOffset = (u64)recorded.offset, dstOffset = (u64)recorded.target_offset, size = (u64)recorded.bytes)
vkCmdCopyBuffer(slot.command, ((VulkanBuffer*)origin.native).buffer, ((VulkanBuffer*)goal.native).buffer, 1, ®ion)
elif recorded.kind == command_fill:
let target = recorded.buffer else trap("a fill lost its buffer")
vkCmdFillBuffer(slot.command, ((VulkanBuffer*)target.native).buffer, (u64)recorded.offset, (u64)recorded.bytes, recorded.value)
elif recorded.kind >= command_build_blas:
try vulkan_encode_build(device, slot, slot.command, recorded, scratch)
else:
try vulkan_encode_dispatch(device, slot.command, pool, recorded)
# Storage images back to sampling, the layout the rest of the device expects.
for recorded in commands:
for bound in recorded.textures:
if let texture = bound: vulkan_transition(slot.command, (VulkanTexture*)texture.native, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
# The results, to whatever is submitted next: frames, reads, other passes.
vulkan_compute_barrier(slot.command, vulkan_compute_stages(device), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)
if pool != 0:
try vulkan_retire(slot, VulkanRetired(kind = vulkan_retired_descriptors, handle = pool))
pool = 0
let serial = try vulkan_end(device, slot)
for recorded in commands:
if let kernel = recorded.kernel: ((VulkanKernel*)kernel.built()).last_use = serial
for bound in recorded.buffers:
if let buffer = bound: ((VulkanBuffer*)buffer.native).last_use = serial
for bound in recorded.textures:
if let texture = bound: ((VulkanTexture*)texture.native).last_use = serial
if let table = recorded.table: vulkan_table_used((VulkanTable*)table.native, serial)
if let accel = recorded.accel: vulkan_accel_used(accel, serial)
for bound in recorded.accels:
if let accel = bound: vulkan_accel_used(accel, serial)
for blas in recorded.instanced:
((VulkanAccel*)blas.native).last_use = serial
if let buffer = recorded.buffer: ((VulkanBuffer*)buffer.native).last_use = serial
if let buffer = recorded.target: ((VulkanBuffer*)buffer.native).last_use = serial
# Buffers kernels may reach by address count as used by every pass.
for buffer in addressed:
((VulkanBuffer*)buffer.native).last_use = serial
# A pass is submitted when the program submits it, so the GPU starts now.
try vulkan_flush(device)
return ComputeSubmitted(serial = serial, native = (void*)(usize)stamped if stamped != 0 else none)
# mark: Timing ================================================================================
func vulkan_query_pool_drop(device: VulkanDevice*, pool: u64):
if pool != 0: vkDestroyQueryPool(device.device, pool, none)
func vulkan_query_pool(device: VulkanDevice*, count: u32) -> u64!:
var info = VkQueryPoolCreateInfo(sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, queryType = VK_QUERY_TYPE_TIMESTAMP, queryCount = count)
var pool: u64 = 0
try vulkan_check(vkCreateQueryPool(device.device, &info, none, &pool), "a Vulkan timestamp query pool could not be created")
return pool
## Finished queries in nanoseconds: the queue's valid bits, scaled by the
## device's tick period.
func vulkan_read_queries(device: VulkanDevice*, pool: u64, first: u32, out: u64[]) -> bool:
let bytes = out.length * 8
if vkGetQueryPoolResults(device.device, pool, first, (u32)out.length, bytes, (void*)out.data, 8, VK_QUERY_RESULT_64_BIT) != VK_SUCCESS:
return false
let mask = 0xffffffffffffffffu64 if device.timestamp_bits >= 64 else (1u64 << (u64)device.timestamp_bits) - 1
for index in 0..<out.length:
out[index] = (u64)((f64)(out[index] & mask) * (f64)device.timestamp_period)
return true
## The start and end of a finished submission timed in the device's queries,
## while its ring slot still holds them.
func vulkan_submission_time(device: VulkanDevice*, serial: u64) -> GpuTime?:
let index = (usize)(serial % (u64)vulkan_slot_limit)
if serial == 0 or device.timing == 0 or device.timing_serial[index] != serial:
return none
var times: u64[2]
if not vulkan_read_queries(device, device.timing, (u32)index * 2, times):
return none
return GpuTime(start = times[0], end = times[1])
func vulkan_compute_stamps(device: VulkanDevice*, native: void*?, out: u64[]) -> !:
let pool = (u64)(usize)(native else error(invalid_geometry, "the pass recorded no timestamps"))
if not vulkan_read_queries(device, pool, 0, out):
error(failed, "Vulkan timestamps could not be read")
## Free a pass's query pool once its submission has finished.
func vulkan_compute_release(device: VulkanDevice*, native: void*, serial: u64):
let item = VulkanRetired(kind = vulkan_retired_queries, handle = (u64)(usize)native)
vulkan_poll(device)
if serial <= device.completed:
vulkan_retired_free(device, item)
return
vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
vulkan_wait(device, serial) catch waited: _ = waited
vulkan_retired_free(device, item)
## After a failure before submission: end the recording unsubmitted and free the pool.
func vulkan_compute_abandon(device: VulkanDevice*, slot: VulkanSlot*, pool: u64):
# The slot's recording is reset when it is next begun.
_ = slot
if pool != 0: vkDestroyDescriptorPool(device.device, pool, none)
## Bind a dispatch's pipeline, a descriptor set of its bindings and its
## uniforms, and dispatch it directly or from its arguments buffer.
func vulkan_encode_dispatch(device: VulkanDevice*, command: u64, pool: u64, recorded: const ComputeCommand*) -> !:
let program = recorded.kernel else trap("a dispatch lost its kernel")
let kernel = (VulkanKernel*)program.built()
var set: u64 = 0
var set_info = VkDescriptorSetAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, descriptorPool = pool, descriptorSetCount = 1, pSetLayouts = &kernel.set_layout)
try vulkan_check(vkAllocateDescriptorSets(device.device, &set_info, &set), "Vulkan compute descriptors could not be allocated")
var infos: VkDescriptorBufferInfo[16]
var images: VkDescriptorImageInfo[16]
var writes: VkWriteDescriptorSet[16]
var count: u32 = 0
for index in 0..<binding_limit:
if program.bindings[index] == (u8)BindingKind.sampler:
images[index] = VkDescriptorImageInfo(sampler = vulkan_sampler_for(device, recorded.filters[index], true))
writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER, pImageInfo = &images[index])
count += 1
continue
if let buffer = recorded.buffers[index]:
let native = (VulkanBuffer*)buffer.native
infos[index] = VkDescriptorBufferInfo(buffer = native.buffer, range = native.bytes)
writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, pBufferInfo = &infos[index])
count += 1
continue
if let accel = recorded.accels[index]:
vulkan_accel_write(device, set, (u32)index, accel)
continue
let texture = recorded.textures[index] else continue
let image = (VulkanTexture*)texture.native
# Storage images are loaded and stored in the general layout; sampled
# ones are read as every draw reads them.
let storage = program.bindings[index] == (u8)BindingKind.storage_image
let layout = VK_IMAGE_LAYOUT_GENERAL if storage else VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
vulkan_transition(command, image, layout)
let sampler = vulkan_sampler_for(device, recorded.filters[index], image.filterable and texture.format != Format.r32_uint)
images[index] = VkDescriptorImageInfo(sampler = 0 if storage else sampler, imageView = image.view, imageLayout = layout)
writes[(usize)count] = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = set, dstBinding = (u32)index, descriptorCount = 1,
descriptorType = vulkan_descriptor_type(program.bindings[index]), pImageInfo = &images[index])
count += 1
if count > 0:
vkUpdateDescriptorSets(device.device, count, &writes[0], 0, none)
vkCmdBindPipeline(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline)
vkCmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline_layout, 0, 1, &set, 0, none)
if let table = recorded.table:
let native = (VulkanTable*)table.native
vulkan_table_ready(command, native)
vkCmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_COMPUTE, kernel.pipeline_layout, 1, 1, &native.set, 0, none)
var uniforms = recorded.uniforms
vkCmdPushConstants(command, kernel.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, (u32)uniform_limit, (void*)&uniforms)
if recorded.kind == command_indirect:
let arguments = recorded.buffer else trap("an indirect dispatch lost its arguments")
vkCmdDispatchIndirect(command, ((VulkanBuffer*)arguments.native).buffer, (u64)recorded.offset)
else:
vkCmdDispatch(command, recorded.groups[0], recorded.groups[1], recorded.groups[2])
# mark: Texture tables ========================================================================
## A texture table: one descriptor set of sampled images, updated after
## binding and partially bound, from a pool of its own; the textures in it,
## made ready for sampling by each dispatch that reads it.
struct VulkanTable:
var pool: u64
var set: u64
var textures: VulkanTexture*?[]
## The newest submission that read the table.
var last_use: u64
## The device's table layout, set 1 of every kernel that reads a table: a
## variable-count array of sampled images, made on first use.
func vulkan_table_layout(device: VulkanDevice*) -> u64!:
if device.table_layout != 0:
return device.table_layout
var binding = VkDescriptorSetLayoutBinding(binding = 0, descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, descriptorCount = device.limits.texture_table_size,
stageFlags = VK_SHADER_STAGE_COMPUTE_BIT)
var flags: u32 = VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT | VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT | VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT
var binding_flags = VkDescriptorSetLayoutBindingFlagsCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, bindingCount = 1, pBindingFlags = &flags)
var info = VkDescriptorSetLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, pNext = (void*)&binding_flags,
flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT, bindingCount = 1, pBindings = &binding)
try vulkan_check(vkCreateDescriptorSetLayout(device.device, &info, none, &device.table_layout), "the Vulkan texture table layout could not be created")
return device.table_layout
func vulkan_table_create(device: VulkanDevice*, capacity: u32) -> VulkanTable*!:
try vulkan_drawing(device)
var layout = try vulkan_table_layout(device)
let textures = try new VulkanTexture*?[(usize)capacity] in memory.heap
let state = try new VulkanTable(textures = textures) in memory.heap
errdefer vulkan_table_destroy(device, state)
var size = VkDescriptorPoolSize(descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, descriptorCount = capacity)
var pool_info = VkDescriptorPoolCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, flags = VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT,
maxSets = 1, poolSizeCount = 1, pPoolSizes = &size)
try vulkan_check(vkCreateDescriptorPool(device.device, &pool_info, none, &state.pool), "a Vulkan texture table could not be created")
var count = capacity
var variable = VkDescriptorSetVariableDescriptorCountAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO, descriptorSetCount = 1, pDescriptorCounts = &count)
var set_info = VkDescriptorSetAllocateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, pNext = (void*)&variable, descriptorPool = state.pool,
descriptorSetCount = 1, pSetLayouts = &layout)
try vulkan_check(vkAllocateDescriptorSets(device.device, &set_info, &state.set), "a Vulkan texture table could not be allocated")
return state
## Point slot `index` at `image` (or, cleared, at the placeholder image, so no
## slot ever names a destroyed view).
func vulkan_table_write(device: VulkanDevice*, table: VulkanTable*, index: u32, image: VulkanTexture*):
var info = VkDescriptorImageInfo(imageView = image.view, imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
var write = VkWriteDescriptorSet(sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, dstSet = table.set, dstBinding = 0, dstArrayElement = index, descriptorCount = 1,
descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, pImageInfo = &info)
vkUpdateDescriptorSets(device.device, 1, &write, 0, none)
func vulkan_table_set(device: VulkanDevice*, table: VulkanTable*, index: u32, texture: VulkanTexture*):
vulkan_table_write(device, table, index, texture)
table.textures[(usize)index] = texture
func vulkan_table_clear(device: VulkanDevice*, table: VulkanTable*, index: u32):
let placeholder = device.placeholder else trap("Vulkan drawing state has no placeholder image")
vulkan_table_write(device, table, index, placeholder)
table.textures[(usize)index] = none
## Every texture in the table in the sampling layout, before a dispatch reads it.
func vulkan_table_ready(command: u64, table: VulkanTable*):
for slot in table.textures:
if let texture = slot: vulkan_transition(command, texture, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
func vulkan_table_used(table: VulkanTable*, serial: u64):
table.last_use = serial
for slot in table.textures:
if let texture = slot: texture.last_use = serial
## Free the table's pool (and its set) once its last submission has finished.
func vulkan_table_destroy(device: VulkanDevice*, table: VulkanTable*):
let item = VulkanRetired(kind = vulkan_retired_descriptors, handle = table.pool)
let serial = table.last_use
free(table.textures) in memory.heap
free(table) in memory.heap
if item.handle == 0: return
vulkan_poll(device)
if serial <= device.completed:
vulkan_retired_free(device, item)
return
vulkan_retire(&device.slots[(usize)(serial % (u64)vulkan_slot_limit)], item) catch failure:
vulkan_wait(device, serial) catch waited: _ = waited
vulkan_retired_free(device, item)