dymokomi/luce-gpu
Source of release 0.5.6
0.5.6 / src / gpu / vulkan / device.lucb
#==============================================================================================
#
# device - Vulkan instance, device and queues
#
# DESCRIPTION:
# Vulkan 1.2 where the loader and device have it, else 1.0: VK_MAKE_API_VERSION(0, 1, 2, 0).
# Optional device features (float atomics) are enabled when the device offers them.
#
#==============================================================================================
let vulkan_api_version_1_0: u32 = 1u32 << 22
let vulkan_api_version_1_1: u32 = (1u32 << 22) | (1u32 << 12)
let vulkan_api_version_1_2: u32 = (1u32 << 22) | (2u32 << 12)
## One render pass pair and pipeline pair per color format in use.
struct VulkanFormatState:
var format: u32
var pass_present: u64
var pass_texture: u64
# The texture pass that draws over the attachment's contents (Texture.frame(keep)).
var pass_load: u64
var overlay: u64
var depth_test: u64
var geometry: u64
# Mesh pipelines by MeshKind (vulkan/mesh.lucb).
var mesh: u64[6]
# The passes of a frame split for scene depth copies, made on first use
# (vulkan_split_pass): opening cleared or loaded, middle, closing to
# presentation or to sampling.
var split: u64[5]
## A free range of a memory page.
struct VulkanRange:
var offset: u64
var size: u64
## One large VkDeviceMemory block that images are carved from, first fit. Its
## free ranges are sorted by offset and never adjacent.
struct VulkanPage:
var memory: u64
var size: u64
var type_index: u32
var live: usize
var ranges: VulkanRange[]
var range_count: usize
var next: VulkanPage*?
## Where a resource's memory lives: a range of a page, or, with no page, a
## dedicated allocation of its own.
struct VulkanAllocation:
var memory: u64
var offset: u64
var size: u64
var page: VulkanPage*?
let vulkan_retired_buffer: u8 = 1
let vulkan_retired_framebuffer: u8 = 2
let vulkan_retired_descriptors: u8 = 3
let vulkan_retired_image: u8 = 4
let vulkan_retired_kernel: u8 = 5
let vulkan_retired_queries: u8 = 6
let vulkan_retired_accel: u8 = 7
## A handle released once the submission holding it has completed. A kernel
## keeps its pipeline in `handle`, its pipeline layout in `view` and its
## descriptor set layout in `extra`.
struct VulkanRetired:
var kind: u8
var handle: u64
var view: u64
var extra: u64
var allocation: VulkanAllocation
## One in-flight submission: its own command pool and buffer, the fence it
## signals, and what it frees when that fence is signalled. `serial` is zero
## while the slot is idle or recording.
struct VulkanSlot:
var pool: u64
var command: u64
var fence: u64
var serial: u64
var retired: VulkanRetired[]
var retired_count: usize
## How many texture submissions may be in flight before recording one waits
## for the oldest.
let vulkan_slot_limit: usize = 32
struct VulkanDevice:
var instance: u64
var physical: u64
var device: u64
# The Vulkan version in use: the lowest of the loader's, the device's and 1.2.
var version: u32
var features: VulkanFeatures
var limits: Limits
# GPU times (vulkan/compute.lucb): the queue's valid timestamp bits, the
# nanoseconds per tick, and two queries per ring slot with the serial
# whose times they hold.
var timestamp_bits: u32
var timestamp_period: f32
var timing: u64
var timing_serial: u64[32]
# Set 1 of kernels reading a texture table (vulkan/compute.lucb).
var table_layout: u64
# Acceleration structure functions and limits (vulkan/accel.lucb), where ray queries are enabled.
var ray: VulkanRay*?
# The open recording: passes, uploads and copies recorded since the last
# submission (vulkan_begin), and how many.
var open: VulkanSlot*?
var open_uses: usize
# The draw-data ring (vulkan/ring.lucb).
var ring: VulkanChunk*?
var ring_current: VulkanChunk*?
var queue: u64
var family: u32
var memory: VkPhysicalDeviceMemoryProperties
# Drawing state shared by surfaces and textures; see vulkan_drawing.
var ready: bool
# The submission ring: serial n uses slots[n % vulkan_slot_limit]. Every
# serial up to `completed` has finished; `submitted` is the newest.
var slots: VulkanSlot[32]
var submitted: u64
var completed: u64
# Image memory pages; see vulkan_image_memory.
var pages: VulkanPage*?
var page_count: usize
# Device-local bytes held for images: pages and dedicated image memory.
var image_bytes: u64
var integrated: bool
var descriptor_layout: u64
var layout: u64
var nearest: u64
var linear: u64
var trilinear: u64
var placeholder: VulkanTexture*?
var fill: u64
var formats: VulkanFormatState[6]
var format_count: usize
var depth_image: u64
var depth_memory: u64
var depth_view: u64
var depth_width: u32
var depth_height: u32
# Counts the texture-pass depth attachments made: a texture's cached
# framebuffer names the one it was made with.
var depth_generation: u64
# The depth copied for draws reading the scene depth (vulkan_depth_copy_for);
# it only grows, and its layout is tracked across submissions.
var depth_copy_image: u64
var depth_copy_memory: u64
var depth_copy_view: u64
var depth_copy_width: u32
var depth_copy_height: u32
var depth_copy_layout: u32
# The staging rings; see vulkan/transfer.lucb.
var reads: VulkanRingSlot[3]
var uploads: VulkanRingSlot[4]
var upload_slot: usize
# Mesh draws: their descriptor and pipeline layouts and a buffer bound
# where a draw leaves one out; see vulkan/mesh.lucb.
var mesh_layout: u64
var mesh_pipeline_layout: u64
var mesh_placeholder: VulkanBuffer*?
func vulkan_check(result: i32, message: str) -> !:
# VK_ERROR_OUT_OF_HOST_MEMORY and VK_ERROR_OUT_OF_DEVICE_MEMORY: room can be made.
if result == -1 or result == -2:
error(out_of_memory, message)
if result != VK_SUCCESS:
error(failed, message)
func vulkan_device_open() -> VulkanDevice*!:
try vulkan_load()
let state = try new VulkanDevice() in memory.heap
errdefer vulkan_device_destroy(state)
# Vulkan 1.2 when the loader knows 1.1 or later (a 1.0 loader refuses higher
# versions); what is used is then capped again by each device's own version.
let requested = vulkan_api_version_1_2 if vulkan_instance_version() >= vulkan_api_version_1_1 else vulkan_api_version_1_0
var application = VkApplicationInfo(sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
pApplicationName = "Luce", pEngineName = "Luce", apiVersion = requested)
let extensions = try vulkan_presentation_extensions()
var instance_info = VkInstanceCreateInfo(sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
pApplicationInfo = &application, enabledExtensionCount = (u32)extensions.length, ppEnabledExtensionNames = (c.str*)extensions.data)
if vkCreateInstance(&instance_info, none, &state.instance) != 0:
error(unavailable, "a Vulkan instance with native presentation is unavailable")
var count: u32 = 0
try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, none), "Vulkan devices could not be enumerated")
if count == 0:
error(unavailable, "no Vulkan device is available")
let devices = try new u64[(usize)count] ---
defer free(devices)
try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, devices.data), "Vulkan devices changed during enumeration")
# a discrete GPU first, then an integrated one, then anything else: on a laptop with
# both, drawing belongs on the one built for it
let order = try new u64[(usize)count] ---
defer free(order)
var ordered: usize
for rank in 0..<3:
for physical in devices[..<(usize)count]:
if vulkan_device_rank(physical) == (u32)rank:
order[ordered] = physical
ordered += 1
for index in 0..<ordered:
let physical = order[index]
var families: u32 = 0
vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, none)
let properties = try new VkQueueFamilyProperties[(usize)families] ---
defer free(properties)
vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, properties.data)
for family in 0..<(usize)families:
if properties[family].queueCount == 0 or (properties[family].queueFlags & VK_QUEUE_GRAPHICS_BIT) == 0 or not vulkan_presentation_support(physical, (u32)family):
continue
var priority: f32 = 1.0
var queue_info = VkDeviceQueueCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
queueFamilyIndex = (u32)family, queueCount = 1, pQueuePriorities = &priority)
let version = vulkan_device_version(physical, requested)
var device_extensions: c.str[5] = [VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
VK_KHR_RAY_QUERY_EXTENSION_NAME, VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME]
var device_info = VkDeviceCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
queueCreateInfoCount = 1, pQueueCreateInfos = &queue_info,
enabledExtensionCount = 1, ppEnabledExtensionNames = &device_extensions[0])
# Optional features the device offers are enabled, only those bits,
# through the features chain: float atomics (an extension), and on
# Vulkan 1.2 buffer addresses, the descriptor indexing texture tables
# need, and ray queries (three extensions).
let offered = try vulkan_offered_features(physical, version)
var atomic_float = VkPhysicalDeviceShaderAtomicFloatFeaturesEXT(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT,
shaderBufferFloat32Atomics = VK_TRUE, shaderBufferFloat32AtomicAdd = VK_TRUE)
let tables = VK_TRUE if offered.texture_tables else VK_FALSE
var vulkan12 = VkPhysicalDeviceVulkan12Features(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
bufferDeviceAddress = VK_TRUE if offered.buffer_addresses else VK_FALSE,
descriptorIndexing = tables, runtimeDescriptorArray = tables, shaderSampledImageArrayNonUniformIndexing = tables,
descriptorBindingPartiallyBound = tables, descriptorBindingVariableDescriptorCount = tables,
descriptorBindingSampledImageUpdateAfterBind = tables)
var accel = VkPhysicalDeviceAccelerationStructureFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR, accelerationStructure = VK_TRUE)
var ray_query = VkPhysicalDeviceRayQueryFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR, pNext = (void*)&accel, rayQuery = VK_TRUE)
var features = VkPhysicalDeviceFeatures2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2)
var chain: void*? = none
# The extensions after the swapchain's, in the array's order:
# float atomics, or (moved down) none, then the ray query three.
if offered.float_atomic_add:
chain = (void*)&atomic_float
device_info.enabledExtensionCount = 2
if offered.ray_query:
if not offered.float_atomic_add:
device_extensions[1] = VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME
device_info.enabledExtensionCount = 4
else:
device_info.enabledExtensionCount = 5
accel.pNext = chain
chain = (void*)&ray_query
if offered.buffer_addresses or offered.texture_tables:
vulkan12.pNext = chain
chain = (void*)&vulkan12
if offered.buffer_addresses:
features.features.shaderInt64 = VK_TRUE
if chain != none:
features.pNext = chain
device_info.pNext = (void*)&features
if vkCreateDevice(physical, &device_info, none, &state.device) != 0:
continue
state.physical = physical
state.version = version
state.features = offered
state.timestamp_bits = properties[family].timestampValidBits
if offered.ray_query:
try vulkan_ray_load(state)
state.limits = vulkan_device_limits(state)
state.integrated = vulkan_device_rank(physical) == 1
state.family = (u32)family
vkGetDeviceQueue(state.device, state.family, 0, &state.queue)
vkGetPhysicalDeviceMemoryProperties(physical, &state.memory)
return state
error(unavailable, "no Vulkan graphics queue supports native presentation")
## The version a device is used at: its own, capped at what the instance requested.
func vulkan_device_version(physical: u64, requested: u32) -> u32:
var properties: VkPhysicalDeviceProperties
vkGetPhysicalDeviceProperties(physical, (void*)&properties)
# Compare major.minor only; the patch level is irrelevant to which calls exist.
let own = properties.apiVersion & 0xfffff000u32
return own if own < requested else requested
## Whether the device offers the extension `wanted`.
func vulkan_has_extension(physical: u64, wanted: str) -> bool!:
var count: u32 = 0
if vkEnumerateDeviceExtensionProperties(physical, none, &count, none) != VK_SUCCESS or count == 0:
return false
let extensions = try new VkExtensionProperties[(usize)count] ---
defer free(extensions)
if vkEnumerateDeviceExtensionProperties(physical, none, &count, extensions.data) != VK_SUCCESS:
return false
for extension in extensions[..<(usize)count]:
var length: usize = 0
while length < extension.extensionName.length and extension.extensionName[length] != 0:
length += 1
if (str)extension.extensionName[..<length] == wanted:
return true
return false
## The optional features luce-gpu enables where a device offers them.
struct VulkanFeatures:
## shaderBufferFloat32AtomicAdd, with VK_EXT_shader_atomic_float.
var float_atomic_add: bool
## bufferDeviceAddress (Vulkan 1.2), with shaderInt64 for the 64-bit
## addresses kernels hold.
var buffer_addresses: bool
## Descriptor indexing for texture tables: runtime arrays of sampled images
## indexed nonuniformly, partially bound, of variable count, updated after
## binding (Vulkan 1.2).
var texture_tables: bool
## Acceleration structures and ray queries (three extensions, Vulkan 1.2,
## with buffer addresses).
var ray_query: bool
func vulkan_offered_features(physical: u64, version: u32) -> VulkanFeatures!:
if version < vulkan_api_version_1_1:
return VulkanFeatures()
let atomics = try vulkan_has_extension(physical, "VK_EXT_shader_atomic_float")
let modern = version >= vulkan_api_version_1_2
let rays = (modern and (try vulkan_has_extension(physical, "VK_KHR_acceleration_structure")) and (try vulkan_has_extension(physical, "VK_KHR_ray_query")) and
(try vulkan_has_extension(physical, "VK_KHR_deferred_host_operations")))
var accel = VkPhysicalDeviceAccelerationStructureFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR)
var ray_query = VkPhysicalDeviceRayQueryFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR, pNext = (void*)&accel)
var atomic_float = VkPhysicalDeviceShaderAtomicFloatFeaturesEXT(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT)
var vulkan12 = VkPhysicalDeviceVulkan12Features(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES)
var features = VkPhysicalDeviceFeatures2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2)
var chain: void*? = none
if atomics: chain = (void*)&atomic_float
if rays:
accel.pNext = chain
chain = (void*)&ray_query
if modern:
vulkan12.pNext = chain
chain = (void*)&vulkan12
features.pNext = chain
vkGetPhysicalDeviceFeatures2(physical, &features)
let tables = (vulkan12.descriptorIndexing != 0 and vulkan12.runtimeDescriptorArray != 0 and vulkan12.shaderSampledImageArrayNonUniformIndexing != 0 and
vulkan12.descriptorBindingPartiallyBound != 0 and vulkan12.descriptorBindingVariableDescriptorCount != 0 and vulkan12.descriptorBindingSampledImageUpdateAfterBind != 0)
let addresses = modern and vulkan12.bufferDeviceAddress != 0 and features.features.shaderInt64 != 0
return VulkanFeatures(float_atomic_add = atomics and atomic_float.shaderBufferFloat32AtomicAdd != 0, buffer_addresses = addresses, texture_tables = modern and tables,
ray_query = rays and addresses and accel.accelerationStructure != 0 and ray_query.rayQuery != 0)
## 0 for a discrete GPU, 1 for an integrated one, 2 for any other kind. Only
## VkPhysicalDeviceProperties' device type is read (bytes 16..20); the bindings
## leave the struct out, and it is under 1 KiB on every platform.
func vulkan_device_rank(physical: u64) -> u32:
var properties: u8[1024] = ---
vkGetPhysicalDeviceProperties(physical, (void*)&properties[0])
let kind = (u32)properties[16] | ((u32)properties[17] << 8) | ((u32)properties[18] << 16) | ((u32)properties[19] << 24)
if kind == 2:
return 0
if kind == 1:
return 1
return 2
## The budget: a discrete GPU's largest device-local heap; an integrated
## GPU's largest heap of any kind, since its textures live in system memory
## (an APU's device-local heap is often a small carve-out, 512 MB, while its
## images spill into the far larger host heap). The image memory held as used.
func vulkan_memory_report(device: VulkanDevice*) -> DeviceMemory:
vulkan_poll(device)
var budget: u64 = 0
for index in 0..<(usize)device.memory.memoryHeapCount:
let heap = device.memory.memoryHeaps[index]
# VK_MEMORY_HEAP_DEVICE_LOCAL_BIT
if (device.integrated or (heap.flags & 1) != 0) and heap.size > budget:
budget = heap.size
return DeviceMemory(budget = budget, used = device.image_bytes, unified = device.integrated)
## Retire finished submissions and count the image memory pages held.
func vulkan_memory_pages(device: VulkanDevice*) -> usize:
vulkan_poll(device)
return device.page_count
func vulkan_device_destroy(state: VulkanDevice*):
if state.device != 0:
_ = vkDeviceWaitIdle(state.device)
vulkan_drawing_destroy(state)
if state.timing != 0: vkDestroyQueryPool(state.device, state.timing, none)
if state.table_layout != 0: vkDestroyDescriptorSetLayout(state.device, state.table_layout, none)
if let ray = state.ray: free(ray) in memory.heap
# Textures still open when the device goes release their pages with it.
while let page = state.pages:
vulkan_page_destroy(state, page)
vkDestroyDevice(state.device, none)
if state.instance != 0:
vkDestroyInstance(state.instance, none)
free(state) in memory.heap
func vulkan_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u32!:
for index in 0..<(usize)device.memory.memoryTypeCount:
if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
return (u32)index
error(unavailable, "no compatible Vulkan memory type is available")
## Whether a memory type has every property in `flags`.
func vulkan_has_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> bool:
for index in 0..<(usize)device.memory.memoryTypeCount:
if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
return true
return false
func vulkan_allocate(device: VulkanDevice*, size: u64, type_index: u32) -> u64!:
var info = VkMemoryAllocateInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, allocationSize = size, memoryTypeIndex = type_index)
var block: u64 = 0
try vulkan_check(vkAllocateMemory(device.device, &info, none, &block), "Vulkan device memory allocation failed")
return block
## A dedicated allocation: staging, vertex and depth memory.
func vulkan_memory(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u64!:
return try vulkan_allocate(device, requirements.size, try vulkan_memory_type(device, requirements, flags))
# mark: Image memory pages ====================================================================
## Pages are 64 MiB, or 256 MiB in a discrete GPU's video memory of more than 8 GiB.
func vulkan_page_size(device: VulkanDevice*, type_index: u32) -> u64:
let heap = device.memory.memoryHeaps[(usize)device.memory.memoryTypes[(usize)type_index].heapIndex]
# An integrated GPU's (an APU's) large heap is the system's memory, which a
# big page would take from everything else for one small texture; a host
# heap (no VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) is too.
let video_memory = not device.integrated and (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0
if video_memory and heap.size > (8u64 << 30):
return 256u64 << 20
return 64u64 << 20
## Device-local memory for an image. Images up to a quarter of a page share
## pages, so a large document makes a few allocations rather than one per tile
## (Windows drivers allow about 4096); bigger ones get memory of their own.
func vulkan_image_memory(device: VulkanDevice*, requirements: VkMemoryRequirements) -> VulkanAllocation!:
let type_index = try vulkan_memory_type(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
let page_size = vulkan_page_size(device, type_index)
if requirements.size > page_size // 4:
let dedicated = VulkanAllocation(memory = try vulkan_allocate(device, requirements.size, type_index), size = requirements.size)
device.image_bytes += requirements.size
return dedicated
var cursor = device.pages
while let page = cursor:
if page.type_index == type_index:
if let offset = try vulkan_page_take(page, requirements.size, requirements.alignment):
return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)
cursor = page.next
let page = try vulkan_page_create(device, type_index, page_size)
let offset = (try vulkan_page_take(page, requirements.size, requirements.alignment)) else trap("a fresh Vulkan memory page cannot hold a quarter of itself")
return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)
func vulkan_page_create(device: VulkanDevice*, type_index: u32, size: u64) -> VulkanPage*!:
let memory_block = try vulkan_allocate(device, size, type_index)
errdefer vkFreeMemory(device.device, memory_block, none)
let page = try new VulkanPage(memory = memory_block, size = size, type_index = type_index) in memory.heap
errdefer free(page) in memory.heap
page.ranges = try new VulkanRange[8] --- in memory.heap
page.ranges[0] = VulkanRange(offset = 0, size = size)
page.range_count = 1
page.next = device.pages
device.pages = page
device.page_count += 1
device.image_bytes += size
return page
func vulkan_page_destroy(device: VulkanDevice*, page: VulkanPage*):
if device.pages == page:
device.pages = page.next
else:
var cursor = device.pages
while let previous = cursor:
if previous.next == page:
previous.next = page.next
break
cursor = previous.next
vkFreeMemory(device.device, page.memory, none)
let size = page.size
free(page.ranges) in memory.heap
free(page) in memory.heap
device.page_count -= 1
device.image_bytes -= size if size <= device.image_bytes else device.image_bytes
## First fit: the aligned offset of `size` bytes taken from a free range, or none
## when no range fits. Free ranges never outnumber live allocations plus one, so
## growing the list to that before taking leaves room for every later release.
func vulkan_page_take(page: VulkanPage*, size: u64, alignment: u64) -> u64?!:
if page.ranges.length < page.live + 2:
let grown = try new VulkanRange[(page.live + 2) * 2] --- in memory.heap
memory.copy(grown, page.ranges, page.range_count)
free(page.ranges) in memory.heap
page.ranges = grown
let align = alignment if alignment > 0 else 1u64
for index in 0..<page.range_count:
let range = page.ranges[index]
let start = (range.offset + align - 1) // align * align
let end = range.offset + range.size
if start + size > end:
continue
let prefix = start - range.offset
let suffix = end - (start + size)
if prefix == 0 and suffix == 0:
for later in index..<(page.range_count - 1):
page.ranges[later] = page.ranges[later + 1]
page.range_count -= 1
elif prefix == 0:
page.ranges[index] = VulkanRange(offset = start + size, size = suffix)
elif suffix == 0:
page.ranges[index].size = prefix
else:
var later = page.range_count
while later > index + 1:
page.ranges[later] = page.ranges[later - 1]
later -= 1
page.ranges[index].size = prefix
page.ranges[index + 1] = VulkanRange(offset = start + size, size = suffix)
page.range_count += 1
page.live += 1
return start
return none
## Return a range to its page, merging it with the free ranges beside it.
func vulkan_page_give(page: VulkanPage*, offset: u64, size: u64):
var index: usize = 0
while index < page.range_count and page.ranges[index].offset < offset:
index += 1
let joins_previous = index > 0 and page.ranges[index - 1].offset + page.ranges[index - 1].size == offset
let joins_next = index < page.range_count and offset + size == page.ranges[index].offset
if joins_previous and joins_next:
page.ranges[index - 1].size += size + page.ranges[index].size
for later in index..<(page.range_count - 1):
page.ranges[later] = page.ranges[later + 1]
page.range_count -= 1
elif joins_previous:
page.ranges[index - 1].size += size
elif joins_next:
page.ranges[index] = VulkanRange(offset = offset, size = size + page.ranges[index].size)
else:
var later = page.range_count
while later > index:
page.ranges[later] = page.ranges[later - 1]
later -= 1
page.ranges[index] = VulkanRange(offset = offset, size = size)
page.range_count += 1
page.live -= 1
## Release memory from `vulkan_image_memory` or a dedicated allocation; a page
## is freed as soon as nothing lives in it.
func vulkan_allocation_free(device: VulkanDevice*, allocation: VulkanAllocation):
if let page = allocation.page:
vulkan_page_give(page, allocation.offset, allocation.size)
if page.live == 0:
vulkan_page_destroy(device, page)
elif allocation.memory != 0:
vkFreeMemory(device.device, allocation.memory, none)
# Dedicated image memory carries its size; staging buffers carry none.
device.image_bytes -= allocation.size if allocation.size <= device.image_bytes else device.image_bytes
# mark: Submissions ===========================================================================
func vulkan_slots_create(device: VulkanDevice*) -> !:
for index in 0..<vulkan_slot_limit:
let slot = &device.slots[index]
var pool_info = VkCommandPoolCreateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, queueFamilyIndex = device.family)
try vulkan_check(vkCreateCommandPool(device.device, &pool_info, none, &slot.pool), "a Vulkan command pool could not be created")
var command_info = VkCommandBufferAllocateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, commandPool = slot.pool, commandBufferCount = 1)
try vulkan_check(vkAllocateCommandBuffers(device.device, &command_info, &slot.command), "a Vulkan command buffer could not be allocated")
var fence_info = VkFenceCreateInfo(sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO)
try vulkan_check(vkCreateFence(device.device, &fence_info, none, &slot.fence), "a Vulkan fence could not be created")
slot.retired = try new VulkanRetired[8] --- in memory.heap
# Two timestamp queries per slot time every submission (vulkan/compute.lucb).
if device.timestamp_bits > 0 and device.timing == 0:
device.timing = try vulkan_query_pool(device, (u32)(vulkan_slot_limit * 2))
## Wait for the device, free everything held, and destroy the ring.
func vulkan_slots_destroy(device: VulkanDevice*):
_ = vkDeviceWaitIdle(device.device)
vulkan_retire_through(device, device.submitted)
for index in 0..<vulkan_slot_limit:
let slot = &device.slots[index]
vulkan_slot_free(device, slot)
if slot.fence != 0: vkDestroyFence(device.device, slot.fence, none)
if slot.pool != 0: vkDestroyCommandPool(device.device, slot.pool, none)
if slot.retired.length > 0: free(slot.retired) in memory.heap
*slot = VulkanSlot()
## Queue `item` for release when `slot`'s submission completes.
func vulkan_retire(slot: VulkanSlot*, item: VulkanRetired) -> !:
if slot.retired_count == slot.retired.length:
let grown = try new VulkanRetired[slot.retired.length * 2 + 8] --- in memory.heap
memory.copy(grown, slot.retired, slot.retired_count)
if slot.retired.length > 0: free(slot.retired) in memory.heap
slot.retired = grown
slot.retired[slot.retired_count] = item
slot.retired_count += 1
## Queue `item` behind the newest submission (the open recording, if any), or
## release it now when none is in flight.
func vulkan_retire_latest(device: VulkanDevice*, item: VulkanRetired):
vulkan_poll(device)
if device.submitted > device.completed or device.open != none:
let slot = device.open else &device.slots[(usize)(device.submitted % (u64)vulkan_slot_limit)]
vulkan_retire(slot, item) catch failure:
_ = failure
# Out of host memory: fall back to waiting rather than leaking or racing.
_ = vkDeviceWaitIdle(device.device)
vulkan_retire_through(device, device.submitted)
vulkan_retired_free(device, item)
return
vulkan_retired_free(device, item)
func vulkan_retired_free(device: VulkanDevice*, item: VulkanRetired):
if item.kind == vulkan_retired_buffer:
vkDestroyBuffer(device.device, item.handle, none)
elif item.kind == vulkan_retired_framebuffer:
vkDestroyFramebuffer(device.device, item.handle, none)
elif item.kind == vulkan_retired_descriptors:
vkDestroyDescriptorPool(device.device, item.handle, none)
elif item.kind == vulkan_retired_image:
if item.view != 0: vkDestroyImageView(device.device, item.view, none)
# A mipmapped texture's view of its first level, for frames.
if item.extra != 0: vkDestroyImageView(device.device, item.extra, none)
if item.handle != 0: vkDestroyImage(device.device, item.handle, none)
elif item.kind == vulkan_retired_queries:
vkDestroyQueryPool(device.device, item.handle, none)
elif item.kind == vulkan_retired_accel:
# An acceleration structure, then the buffer holding it.
if item.handle != 0: vulkan_ray_destroy(device, item.handle)
if item.view != 0: vkDestroyBuffer(device.device, item.view, none)
elif item.kind == vulkan_retired_kernel:
if item.handle != 0: vkDestroyPipeline(device.device, item.handle, none)
if item.view != 0: vkDestroyPipelineLayout(device.device, item.view, none)
if item.extra != 0: vkDestroyDescriptorSetLayout(device.device, item.extra, none)
vulkan_allocation_free(device, item.allocation)
func vulkan_slot_free(device: VulkanDevice*, slot: VulkanSlot*):
for index in 0..<slot.retired_count:
vulkan_retired_free(device, slot.retired[index])
slot.retired_count = 0
slot.serial = 0
## Mark every serial up to `serial` complete and free what they held. Serials
## complete in order: a fence signalled by a submission covers every earlier
## submission on the queue.
func vulkan_retire_through(device: VulkanDevice*, serial: u64):
while device.completed < serial:
device.completed += 1
vulkan_slot_free(device, &device.slots[(usize)(device.completed % (u64)vulkan_slot_limit)])
## Retire the submissions that have finished, oldest first, without waiting.
func vulkan_poll(device: VulkanDevice*):
while device.completed < device.submitted:
let slot = &device.slots[(usize)((device.completed + 1) % (u64)vulkan_slot_limit)]
if vkWaitForFences(device.device, 1, &slot.fence, 1, 0) != VK_SUCCESS:
return
vulkan_retire_through(device, device.completed + 1)
## Wait until submission `serial` has completed, and retire it and every earlier
## one; the open recording's serial is submitted first.
func vulkan_wait(device: VulkanDevice*, serial: u64) -> !:
if serial <= device.completed:
return
if serial > device.submitted:
try vulkan_flush(device)
let slot = &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]
let result = vkWaitForFences(device.device, 1, &slot.fence, 1, 0xffffffffffffffffu64)
# VK_ERROR_DEVICE_LOST: the work failed on the GPU.
if result == -4:
error(execution_failed, "the GPU failed while running submitted work")
try vulkan_check(result, "Vulkan execution failed")
vulkan_retire_through(device, serial)
## Everything submitted before a host wait on the whole queue has completed.
func vulkan_idle(device: VulkanDevice*):
vulkan_retire_through(device, device.submitted)
## How many recordings (passes, uploads, copies) the open recording takes before
## it is submitted on its own: what bounds the latency and the draw-data ring for
## a program that renders offscreen without presenting or reading.
let vulkan_open_limit: usize = 64
## Record into the next submission: the open recording, or a new one. When the
## ring is full this waits for the oldest submission; the serial the recording
## will get is `device.submitted + 1`. A new recording starts with the
## submission's start time.
func vulkan_begin(device: VulkanDevice*) -> VulkanSlot*!:
if let open = device.open:
device.open_uses += 1
return open
let slot = try vulkan_reserve(device)
try vulkan_check(vkResetCommandPool(device.device, slot.pool, 0), "a Vulkan command pool could not be reset")
var begin = VkCommandBufferBeginInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT)
try vulkan_check(vkBeginCommandBuffer(slot.command, &begin), "Vulkan recording could not begin")
vulkan_time_start(device, slot.command)
device.open = slot
device.open_uses = 1
return slot
## The ring slot the next submission takes, once free, with its command
## buffer: a frame records into it directly. That slot is the open recording's
## when there is one, so a frame submits that recording first.
func vulkan_reserve(device: VulkanDevice*) -> VulkanSlot*!:
if device.open != none: trap("a Vulkan submission was reserved over the open recording")
vulkan_poll(device)
let serial = device.submitted + 1
if serial > (u64)vulkan_slot_limit:
try vulkan_wait(device, serial - (u64)vulkan_slot_limit)
return &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]
## Record the next submission's start time into `command`.
func vulkan_time_start(device: VulkanDevice*, command: u64):
if device.timing == 0: return
let first = (u32)((device.submitted + 1) % (u64)vulkan_slot_limit) * 2
vkCmdResetQueryPool(command, device.timing, first, 2)
vkCmdWriteTimestamp(command, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, device.timing, first)
## Record the next submission's end time into `command`, after all its work.
func vulkan_time_end(device: VulkanDevice*, command: u64):
if device.timing == 0: return
let first = (u32)((device.submitted + 1) % (u64)vulkan_slot_limit) * 2
vkCmdWriteTimestamp(command, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, device.timing, first + 1)
## Submit `submit` with `slot`'s fence as the next serial, and return it.
func vulkan_submit_reserved(device: VulkanDevice*, slot: VulkanSlot*, submit: VkSubmitInfo*) -> u64!:
try vulkan_check(vkResetFences(device.device, 1, &slot.fence), "a Vulkan fence could not be reset")
try vulkan_check(vkQueueSubmit(device.queue, 1, submit, slot.fence), "Vulkan submission failed")
device.submitted += 1
slot.serial = device.submitted
if device.timing != 0: device.timing_serial[(usize)(device.submitted % (u64)vulkan_slot_limit)] = device.submitted
return device.submitted
## After a failed submission: drain the device and free what the slot held.
func vulkan_abandon(device: VulkanDevice*, slot: VulkanSlot*):
_ = vkDeviceWaitIdle(device.device)
vulkan_retire_through(device, device.submitted)
vulkan_slot_free(device, slot)
## End a piece of recording begun by `vulkan_begin` and return the serial its
## submission gets. The recording stays open for more, and is submitted at a
## surface's present, before a read or compute pass, when a wait asks for its
## serial, or after `vulkan_open_limit` pieces; later work is ordered after it.
func vulkan_end(device: VulkanDevice*, slot: VulkanSlot*) -> u64!:
_ = slot
let serial = device.submitted + 1
if device.open_uses >= vulkan_open_limit:
try vulkan_flush(device)
return serial
## Submit the open recording, if any, without waiting. If submission fails, the
## device is drained and what the recording held is freed.
func vulkan_flush(device: VulkanDevice*) -> !:
let slot = device.open else return
device.open = none
device.open_uses = 0
errdefer vulkan_abandon(device, slot)
vulkan_time_end(device, slot.command)
try vulkan_check(vkEndCommandBuffer(slot.command), "Vulkan recording could not finish")
var submit = VkSubmitInfo(sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, commandBufferCount = 1, pCommandBuffers = &slot.command)
_ = try vulkan_submit_reserved(device, slot, &submit)
vulkan_ring_trim(device)