Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.6

0.5.6 / src / gpu / vulkan / device.lucb

#==============================================================================================
#
#   device - Vulkan instance, device and queues
#
#   DESCRIPTION:
#       Vulkan 1.2 where the loader and device have it, else 1.0: VK_MAKE_API_VERSION(0, 1, 2, 0).
#       Optional device features (float atomics) are enabled when the device offers them.
#
#==============================================================================================

let vulkan_api_version_1_0: u32 = 1u32 << 22
let vulkan_api_version_1_1: u32 = (1u32 << 22) | (1u32 << 12)
let vulkan_api_version_1_2: u32 = (1u32 << 22) | (2u32 << 12)

## One render pass pair and pipeline pair per color format in use.
struct VulkanFormatState:
    var format: u32
    var pass_present: u64
    var pass_texture: u64
    # The texture pass that draws over the attachment's contents (Texture.frame(keep)).
    var pass_load: u64
    var overlay: u64
    var depth_test: u64
    var geometry: u64
    # Mesh pipelines by MeshKind (vulkan/mesh.lucb).
    var mesh: u64[6]
    # The passes of a frame split for scene depth copies, made on first use
    # (vulkan_split_pass): opening cleared or loaded, middle, closing to
    # presentation or to sampling.
    var split: u64[5]

## A free range of a memory page.
struct VulkanRange:
    var offset: u64
    var size: u64

## One large VkDeviceMemory block that images are carved from, first fit. Its
## free ranges are sorted by offset and never adjacent.
struct VulkanPage:
    var memory: u64
    var size: u64
    var type_index: u32
    var live: usize
    var ranges: VulkanRange[]
    var range_count: usize
    var next: VulkanPage*?

## Where a resource's memory lives: a range of a page, or, with no page, a
## dedicated allocation of its own.
struct VulkanAllocation:
    var memory: u64
    var offset: u64
    var size: u64
    var page: VulkanPage*?

let vulkan_retired_buffer: u8 = 1
let vulkan_retired_framebuffer: u8 = 2
let vulkan_retired_descriptors: u8 = 3
let vulkan_retired_image: u8 = 4
let vulkan_retired_kernel: u8 = 5
let vulkan_retired_queries: u8 = 6
let vulkan_retired_accel: u8 = 7

## A handle released once the submission holding it has completed. A kernel
## keeps its pipeline in `handle`, its pipeline layout in `view` and its
## descriptor set layout in `extra`.
struct VulkanRetired:
    var kind: u8
    var handle: u64
    var view: u64
    var extra: u64
    var allocation: VulkanAllocation

## One in-flight submission: its own command pool and buffer, the fence it
## signals, and what it frees when that fence is signalled. `serial` is zero
## while the slot is idle or recording.
struct VulkanSlot:
    var pool: u64
    var command: u64
    var fence: u64
    var serial: u64
    var retired: VulkanRetired[]
    var retired_count: usize

## How many texture submissions may be in flight before recording one waits
## for the oldest.
let vulkan_slot_limit: usize = 32

struct VulkanDevice:
    var instance: u64
    var physical: u64
    var device: u64
    # The Vulkan version in use: the lowest of the loader's, the device's and 1.2.
    var version: u32
    var features: VulkanFeatures
    var limits: Limits
    # GPU times (vulkan/compute.lucb): the queue's valid timestamp bits, the
    # nanoseconds per tick, and two queries per ring slot with the serial
    # whose times they hold.
    var timestamp_bits: u32
    var timestamp_period: f32
    var timing: u64
    var timing_serial: u64[32]
    # Set 1 of kernels reading a texture table (vulkan/compute.lucb).
    var table_layout: u64
    # Acceleration structure functions and limits (vulkan/accel.lucb), where ray queries are enabled.
    var ray: VulkanRay*?
    # The open recording: passes, uploads and copies recorded since the last
    # submission (vulkan_begin), and how many.
    var open: VulkanSlot*?
    var open_uses: usize
    # The draw-data ring (vulkan/ring.lucb).
    var ring: VulkanChunk*?
    var ring_current: VulkanChunk*?
    var queue: u64
    var family: u32
    var memory: VkPhysicalDeviceMemoryProperties
    # Drawing state shared by surfaces and textures; see vulkan_drawing.
    var ready: bool
    # The submission ring: serial n uses slots[n % vulkan_slot_limit]. Every
    # serial up to `completed` has finished; `submitted` is the newest.
    var slots: VulkanSlot[32]
    var submitted: u64
    var completed: u64
    # Image memory pages; see vulkan_image_memory.
    var pages: VulkanPage*?
    var page_count: usize
    # Device-local bytes held for images: pages and dedicated image memory.
    var image_bytes: u64
    var integrated: bool
    var descriptor_layout: u64
    var layout: u64
    var nearest: u64
    var linear: u64
    var trilinear: u64
    var placeholder: VulkanTexture*?
    var fill: u64
    var formats: VulkanFormatState[6]
    var format_count: usize
    var depth_image: u64
    var depth_memory: u64
    var depth_view: u64
    var depth_width: u32
    var depth_height: u32
    # Counts the texture-pass depth attachments made: a texture's cached
    # framebuffer names the one it was made with.
    var depth_generation: u64
    # The depth copied for draws reading the scene depth (vulkan_depth_copy_for);
    # it only grows, and its layout is tracked across submissions.
    var depth_copy_image: u64
    var depth_copy_memory: u64
    var depth_copy_view: u64
    var depth_copy_width: u32
    var depth_copy_height: u32
    var depth_copy_layout: u32
    # The staging rings; see vulkan/transfer.lucb.
    var reads: VulkanRingSlot[3]
    var uploads: VulkanRingSlot[4]
    var upload_slot: usize
    # Mesh draws: their descriptor and pipeline layouts and a buffer bound
    # where a draw leaves one out; see vulkan/mesh.lucb.
    var mesh_layout: u64
    var mesh_pipeline_layout: u64
    var mesh_placeholder: VulkanBuffer*?

func vulkan_check(result: i32, message: str) -> !:
    # VK_ERROR_OUT_OF_HOST_MEMORY and VK_ERROR_OUT_OF_DEVICE_MEMORY: room can be made.
    if result == -1 or result == -2:
        error(out_of_memory, message)
    if result != VK_SUCCESS:
        error(failed, message)

func vulkan_device_open() -> VulkanDevice*!:
    try vulkan_load()
    let state = try new VulkanDevice() in memory.heap
    errdefer vulkan_device_destroy(state)
    # Vulkan 1.2 when the loader knows 1.1 or later (a 1.0 loader refuses higher
    # versions); what is used is then capped again by each device's own version.
    let requested = vulkan_api_version_1_2 if vulkan_instance_version() >= vulkan_api_version_1_1 else vulkan_api_version_1_0
    var application = VkApplicationInfo(sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
                                        pApplicationName = "Luce", pEngineName = "Luce", apiVersion = requested)
    let extensions = try vulkan_presentation_extensions()
    var instance_info = VkInstanceCreateInfo(sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
                                             pApplicationInfo = &application, enabledExtensionCount = (u32)extensions.length, ppEnabledExtensionNames = (c.str*)extensions.data)
    if vkCreateInstance(&instance_info, none, &state.instance) != 0:
        error(unavailable, "a Vulkan instance with native presentation is unavailable")
    var count: u32 = 0
    try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, none), "Vulkan devices could not be enumerated")
    if count == 0:
        error(unavailable, "no Vulkan device is available")
    let devices = try new u64[(usize)count] ---
    defer free(devices)
    try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, devices.data), "Vulkan devices changed during enumeration")
    # a discrete GPU first, then an integrated one, then anything else: on a laptop with
    # both, drawing belongs on the one built for it
    let order = try new u64[(usize)count] ---
    defer free(order)
    var ordered: usize
    for rank in 0..<3:
        for physical in devices[..<(usize)count]:
            if vulkan_device_rank(physical) == (u32)rank:
                order[ordered] = physical
                ordered += 1
    for index in 0..<ordered:
        let physical = order[index]
        var families: u32 = 0
        vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, none)
        let properties = try new VkQueueFamilyProperties[(usize)families] ---
        defer free(properties)
        vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, properties.data)
        for family in 0..<(usize)families:
            if properties[family].queueCount == 0 or (properties[family].queueFlags & VK_QUEUE_GRAPHICS_BIT) == 0 or not vulkan_presentation_support(physical, (u32)family):
                continue
            var priority: f32 = 1.0
            var queue_info = VkDeviceQueueCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
                                                     queueFamilyIndex = (u32)family, queueCount = 1, pQueuePriorities = &priority)
            let version = vulkan_device_version(physical, requested)
            var device_extensions: c.str[5] = [VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
                                               VK_KHR_RAY_QUERY_EXTENSION_NAME, VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME]
            var device_info = VkDeviceCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
                                                 queueCreateInfoCount = 1, pQueueCreateInfos = &queue_info,
                                                 enabledExtensionCount = 1, ppEnabledExtensionNames = &device_extensions[0])
            # Optional features the device offers are enabled, only those bits,
            # through the features chain: float atomics (an extension), and on
            # Vulkan 1.2 buffer addresses, the descriptor indexing texture tables
            # need, and ray queries (three extensions).
            let offered = try vulkan_offered_features(physical, version)
            var atomic_float = VkPhysicalDeviceShaderAtomicFloatFeaturesEXT(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT,
                                                                            shaderBufferFloat32Atomics = VK_TRUE, shaderBufferFloat32AtomicAdd = VK_TRUE)
            let tables = VK_TRUE if offered.texture_tables else VK_FALSE
            var vulkan12 = VkPhysicalDeviceVulkan12Features(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
                                                            bufferDeviceAddress = VK_TRUE if offered.buffer_addresses else VK_FALSE,
                                                            descriptorIndexing = tables, runtimeDescriptorArray = tables, shaderSampledImageArrayNonUniformIndexing = tables,
                                                            descriptorBindingPartiallyBound = tables, descriptorBindingVariableDescriptorCount = tables,
                                                            descriptorBindingSampledImageUpdateAfterBind = tables)
            var accel = VkPhysicalDeviceAccelerationStructureFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR, accelerationStructure = VK_TRUE)
            var ray_query = VkPhysicalDeviceRayQueryFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR, pNext = (void*)&accel, rayQuery = VK_TRUE)
            var features = VkPhysicalDeviceFeatures2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2)
            var chain: void*? = none
            # The extensions after the swapchain's, in the array's order:
            # float atomics, or (moved down) none, then the ray query three.
            if offered.float_atomic_add:
                chain = (void*)&atomic_float
                device_info.enabledExtensionCount = 2
            if offered.ray_query:
                if not offered.float_atomic_add:
                    device_extensions[1] = VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME
                    device_info.enabledExtensionCount = 4
                else:
                    device_info.enabledExtensionCount = 5
                accel.pNext = chain
                chain = (void*)&ray_query
            if offered.buffer_addresses or offered.texture_tables:
                vulkan12.pNext = chain
                chain = (void*)&vulkan12
            if offered.buffer_addresses:
                features.features.shaderInt64 = VK_TRUE
            if chain != none:
                features.pNext = chain
                device_info.pNext = (void*)&features
            if vkCreateDevice(physical, &device_info, none, &state.device) != 0:
                continue
            state.physical = physical
            state.version = version
            state.features = offered
            state.timestamp_bits = properties[family].timestampValidBits
            if offered.ray_query:
                try vulkan_ray_load(state)
            state.limits = vulkan_device_limits(state)
            state.integrated = vulkan_device_rank(physical) == 1
            state.family = (u32)family
            vkGetDeviceQueue(state.device, state.family, 0, &state.queue)
            vkGetPhysicalDeviceMemoryProperties(physical, &state.memory)
            return state
    error(unavailable, "no Vulkan graphics queue supports native presentation")

## The version a device is used at: its own, capped at what the instance requested.
func vulkan_device_version(physical: u64, requested: u32) -> u32:
    var properties: VkPhysicalDeviceProperties
    vkGetPhysicalDeviceProperties(physical, (void*)&properties)
    # Compare major.minor only; the patch level is irrelevant to which calls exist.
    let own = properties.apiVersion & 0xfffff000u32
    return own if own < requested else requested

## Whether the device offers the extension `wanted`.
func vulkan_has_extension(physical: u64, wanted: str) -> bool!:
    var count: u32 = 0
    if vkEnumerateDeviceExtensionProperties(physical, none, &count, none) != VK_SUCCESS or count == 0:
        return false
    let extensions = try new VkExtensionProperties[(usize)count] ---
    defer free(extensions)
    if vkEnumerateDeviceExtensionProperties(physical, none, &count, extensions.data) != VK_SUCCESS:
        return false
    for extension in extensions[..<(usize)count]:
        var length: usize = 0
        while length < extension.extensionName.length and extension.extensionName[length] != 0:
            length += 1
        if (str)extension.extensionName[..<length] == wanted:
            return true
    return false

## The optional features luce-gpu enables where a device offers them.
struct VulkanFeatures:
    ## shaderBufferFloat32AtomicAdd, with VK_EXT_shader_atomic_float.
    var float_atomic_add: bool
    ## bufferDeviceAddress (Vulkan 1.2), with shaderInt64 for the 64-bit
    ## addresses kernels hold.
    var buffer_addresses: bool
    ## Descriptor indexing for texture tables: runtime arrays of sampled images
    ## indexed nonuniformly, partially bound, of variable count, updated after
    ## binding (Vulkan 1.2).
    var texture_tables: bool
    ## Acceleration structures and ray queries (three extensions, Vulkan 1.2,
    ## with buffer addresses).
    var ray_query: bool

func vulkan_offered_features(physical: u64, version: u32) -> VulkanFeatures!:
    if version < vulkan_api_version_1_1:
        return VulkanFeatures()
    let atomics = try vulkan_has_extension(physical, "VK_EXT_shader_atomic_float")
    let modern = version >= vulkan_api_version_1_2
    let rays = (modern and (try vulkan_has_extension(physical, "VK_KHR_acceleration_structure")) and (try vulkan_has_extension(physical, "VK_KHR_ray_query")) and
                (try vulkan_has_extension(physical, "VK_KHR_deferred_host_operations")))
    var accel = VkPhysicalDeviceAccelerationStructureFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR)
    var ray_query = VkPhysicalDeviceRayQueryFeaturesKHR(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR, pNext = (void*)&accel)
    var atomic_float = VkPhysicalDeviceShaderAtomicFloatFeaturesEXT(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT)
    var vulkan12 = VkPhysicalDeviceVulkan12Features(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES)
    var features = VkPhysicalDeviceFeatures2(sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2)
    var chain: void*? = none
    if atomics: chain = (void*)&atomic_float
    if rays:
        accel.pNext = chain
        chain = (void*)&ray_query
    if modern:
        vulkan12.pNext = chain
        chain = (void*)&vulkan12
    features.pNext = chain
    vkGetPhysicalDeviceFeatures2(physical, &features)
    let tables = (vulkan12.descriptorIndexing != 0 and vulkan12.runtimeDescriptorArray != 0 and vulkan12.shaderSampledImageArrayNonUniformIndexing != 0 and
                  vulkan12.descriptorBindingPartiallyBound != 0 and vulkan12.descriptorBindingVariableDescriptorCount != 0 and vulkan12.descriptorBindingSampledImageUpdateAfterBind != 0)
    let addresses = modern and vulkan12.bufferDeviceAddress != 0 and features.features.shaderInt64 != 0
    return VulkanFeatures(float_atomic_add = atomics and atomic_float.shaderBufferFloat32AtomicAdd != 0, buffer_addresses = addresses, texture_tables = modern and tables,
                          ray_query = rays and addresses and accel.accelerationStructure != 0 and ray_query.rayQuery != 0)

## 0 for a discrete GPU, 1 for an integrated one, 2 for any other kind. Only
## VkPhysicalDeviceProperties' device type is read (bytes 16..20); the bindings
## leave the struct out, and it is under 1 KiB on every platform.
func vulkan_device_rank(physical: u64) -> u32:
    var properties: u8[1024] = ---
    vkGetPhysicalDeviceProperties(physical, (void*)&properties[0])
    let kind = (u32)properties[16] | ((u32)properties[17] << 8) | ((u32)properties[18] << 16) | ((u32)properties[19] << 24)
    if kind == 2:
        return 0
    if kind == 1:
        return 1
    return 2

## The budget: a discrete GPU's largest device-local heap; an integrated
## GPU's largest heap of any kind, since its textures live in system memory
## (an APU's device-local heap is often a small carve-out, 512 MB, while its
## images spill into the far larger host heap). The image memory held as used.
func vulkan_memory_report(device: VulkanDevice*) -> DeviceMemory:
    vulkan_poll(device)
    var budget: u64 = 0
    for index in 0..<(usize)device.memory.memoryHeapCount:
        let heap = device.memory.memoryHeaps[index]
        # VK_MEMORY_HEAP_DEVICE_LOCAL_BIT
        if (device.integrated or (heap.flags & 1) != 0) and heap.size > budget:
            budget = heap.size
    return DeviceMemory(budget = budget, used = device.image_bytes, unified = device.integrated)

## Retire finished submissions and count the image memory pages held.
func vulkan_memory_pages(device: VulkanDevice*) -> usize:
    vulkan_poll(device)
    return device.page_count

func vulkan_device_destroy(state: VulkanDevice*):
    if state.device != 0:
        _ = vkDeviceWaitIdle(state.device)
        vulkan_drawing_destroy(state)
        if state.timing != 0: vkDestroyQueryPool(state.device, state.timing, none)
        if state.table_layout != 0: vkDestroyDescriptorSetLayout(state.device, state.table_layout, none)
        if let ray = state.ray: free(ray) in memory.heap
        # Textures still open when the device goes release their pages with it.
        while let page = state.pages:
            vulkan_page_destroy(state, page)
        vkDestroyDevice(state.device, none)
    if state.instance != 0:
        vkDestroyInstance(state.instance, none)
    free(state) in memory.heap

func vulkan_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u32!:
    for index in 0..<(usize)device.memory.memoryTypeCount:
        if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
            return (u32)index
    error(unavailable, "no compatible Vulkan memory type is available")

## Whether a memory type has every property in `flags`.
func vulkan_has_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> bool:
    for index in 0..<(usize)device.memory.memoryTypeCount:
        if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
            return true
    return false

func vulkan_allocate(device: VulkanDevice*, size: u64, type_index: u32) -> u64!:
    var info = VkMemoryAllocateInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, allocationSize = size, memoryTypeIndex = type_index)
    var block: u64 = 0
    try vulkan_check(vkAllocateMemory(device.device, &info, none, &block), "Vulkan device memory allocation failed")
    return block

## A dedicated allocation: staging, vertex and depth memory.
func vulkan_memory(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u64!:
    return try vulkan_allocate(device, requirements.size, try vulkan_memory_type(device, requirements, flags))

# mark: Image memory pages ====================================================================

## Pages are 64 MiB, or 256 MiB in a discrete GPU's video memory of more than 8 GiB.
func vulkan_page_size(device: VulkanDevice*, type_index: u32) -> u64:
    let heap = device.memory.memoryHeaps[(usize)device.memory.memoryTypes[(usize)type_index].heapIndex]
    # An integrated GPU's (an APU's) large heap is the system's memory, which a
    # big page would take from everything else for one small texture; a host
    # heap (no VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) is too.
    let video_memory = not device.integrated and (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0
    if video_memory and heap.size > (8u64 << 30):
        return 256u64 << 20
    return 64u64 << 20

## Device-local memory for an image. Images up to a quarter of a page share
## pages, so a large document makes a few allocations rather than one per tile
## (Windows drivers allow about 4096); bigger ones get memory of their own.
func vulkan_image_memory(device: VulkanDevice*, requirements: VkMemoryRequirements) -> VulkanAllocation!:
    let type_index = try vulkan_memory_type(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
    let page_size = vulkan_page_size(device, type_index)
    if requirements.size > page_size // 4:
        let dedicated = VulkanAllocation(memory = try vulkan_allocate(device, requirements.size, type_index), size = requirements.size)
        device.image_bytes += requirements.size
        return dedicated
    var cursor = device.pages
    while let page = cursor:
        if page.type_index == type_index:
            if let offset = try vulkan_page_take(page, requirements.size, requirements.alignment):
                return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)
        cursor = page.next
    let page = try vulkan_page_create(device, type_index, page_size)
    let offset = (try vulkan_page_take(page, requirements.size, requirements.alignment)) else trap("a fresh Vulkan memory page cannot hold a quarter of itself")
    return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)

func vulkan_page_create(device: VulkanDevice*, type_index: u32, size: u64) -> VulkanPage*!:
    let memory_block = try vulkan_allocate(device, size, type_index)
    errdefer vkFreeMemory(device.device, memory_block, none)
    let page = try new VulkanPage(memory = memory_block, size = size, type_index = type_index) in memory.heap
    errdefer free(page) in memory.heap
    page.ranges = try new VulkanRange[8] --- in memory.heap
    page.ranges[0] = VulkanRange(offset = 0, size = size)
    page.range_count = 1
    page.next = device.pages
    device.pages = page
    device.page_count += 1
    device.image_bytes += size
    return page

func vulkan_page_destroy(device: VulkanDevice*, page: VulkanPage*):
    if device.pages == page:
        device.pages = page.next
    else:
        var cursor = device.pages
        while let previous = cursor:
            if previous.next == page:
                previous.next = page.next
                break
            cursor = previous.next
    vkFreeMemory(device.device, page.memory, none)
    let size = page.size
    free(page.ranges) in memory.heap
    free(page) in memory.heap
    device.page_count -= 1
    device.image_bytes -= size if size <= device.image_bytes else device.image_bytes

## First fit: the aligned offset of `size` bytes taken from a free range, or none
## when no range fits. Free ranges never outnumber live allocations plus one, so
## growing the list to that before taking leaves room for every later release.
func vulkan_page_take(page: VulkanPage*, size: u64, alignment: u64) -> u64?!:
    if page.ranges.length < page.live + 2:
        let grown = try new VulkanRange[(page.live + 2) * 2] --- in memory.heap
        memory.copy(grown, page.ranges, page.range_count)
        free(page.ranges) in memory.heap
        page.ranges = grown
    let align = alignment if alignment > 0 else 1u64
    for index in 0..<page.range_count:
        let range = page.ranges[index]
        let start = (range.offset + align - 1) // align * align
        let end = range.offset + range.size
        if start + size > end:
            continue
        let prefix = start - range.offset
        let suffix = end - (start + size)
        if prefix == 0 and suffix == 0:
            for later in index..<(page.range_count - 1):
                page.ranges[later] = page.ranges[later + 1]
            page.range_count -= 1
        elif prefix == 0:
            page.ranges[index] = VulkanRange(offset = start + size, size = suffix)
        elif suffix == 0:
            page.ranges[index].size = prefix
        else:
            var later = page.range_count
            while later > index + 1:
                page.ranges[later] = page.ranges[later - 1]
                later -= 1
            page.ranges[index].size = prefix
            page.ranges[index + 1] = VulkanRange(offset = start + size, size = suffix)
            page.range_count += 1
        page.live += 1
        return start
    return none

## Return a range to its page, merging it with the free ranges beside it.
func vulkan_page_give(page: VulkanPage*, offset: u64, size: u64):
    var index: usize = 0
    while index < page.range_count and page.ranges[index].offset < offset:
        index += 1
    let joins_previous = index > 0 and page.ranges[index - 1].offset + page.ranges[index - 1].size == offset
    let joins_next = index < page.range_count and offset + size == page.ranges[index].offset
    if joins_previous and joins_next:
        page.ranges[index - 1].size += size + page.ranges[index].size
        for later in index..<(page.range_count - 1):
            page.ranges[later] = page.ranges[later + 1]
        page.range_count -= 1
    elif joins_previous:
        page.ranges[index - 1].size += size
    elif joins_next:
        page.ranges[index] = VulkanRange(offset = offset, size = size + page.ranges[index].size)
    else:
        var later = page.range_count
        while later > index:
            page.ranges[later] = page.ranges[later - 1]
            later -= 1
        page.ranges[index] = VulkanRange(offset = offset, size = size)
        page.range_count += 1
    page.live -= 1

## Release memory from `vulkan_image_memory` or a dedicated allocation; a page
## is freed as soon as nothing lives in it.
func vulkan_allocation_free(device: VulkanDevice*, allocation: VulkanAllocation):
    if let page = allocation.page:
        vulkan_page_give(page, allocation.offset, allocation.size)
        if page.live == 0:
            vulkan_page_destroy(device, page)
    elif allocation.memory != 0:
        vkFreeMemory(device.device, allocation.memory, none)
        # Dedicated image memory carries its size; staging buffers carry none.
        device.image_bytes -= allocation.size if allocation.size <= device.image_bytes else device.image_bytes

# mark: Submissions ===========================================================================

func vulkan_slots_create(device: VulkanDevice*) -> !:
    for index in 0..<vulkan_slot_limit:
        let slot = &device.slots[index]
        var pool_info = VkCommandPoolCreateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, queueFamilyIndex = device.family)
        try vulkan_check(vkCreateCommandPool(device.device, &pool_info, none, &slot.pool), "a Vulkan command pool could not be created")
        var command_info = VkCommandBufferAllocateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, commandPool = slot.pool, commandBufferCount = 1)
        try vulkan_check(vkAllocateCommandBuffers(device.device, &command_info, &slot.command), "a Vulkan command buffer could not be allocated")
        var fence_info = VkFenceCreateInfo(sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO)
        try vulkan_check(vkCreateFence(device.device, &fence_info, none, &slot.fence), "a Vulkan fence could not be created")
        slot.retired = try new VulkanRetired[8] --- in memory.heap
    # Two timestamp queries per slot time every submission (vulkan/compute.lucb).
    if device.timestamp_bits > 0 and device.timing == 0:
        device.timing = try vulkan_query_pool(device, (u32)(vulkan_slot_limit * 2))

## Wait for the device, free everything held, and destroy the ring.
func vulkan_slots_destroy(device: VulkanDevice*):
    _ = vkDeviceWaitIdle(device.device)
    vulkan_retire_through(device, device.submitted)
    for index in 0..<vulkan_slot_limit:
        let slot = &device.slots[index]
        vulkan_slot_free(device, slot)
        if slot.fence != 0: vkDestroyFence(device.device, slot.fence, none)
        if slot.pool != 0: vkDestroyCommandPool(device.device, slot.pool, none)
        if slot.retired.length > 0: free(slot.retired) in memory.heap
        *slot = VulkanSlot()

## Queue `item` for release when `slot`'s submission completes.
func vulkan_retire(slot: VulkanSlot*, item: VulkanRetired) -> !:
    if slot.retired_count == slot.retired.length:
        let grown = try new VulkanRetired[slot.retired.length * 2 + 8] --- in memory.heap
        memory.copy(grown, slot.retired, slot.retired_count)
        if slot.retired.length > 0: free(slot.retired) in memory.heap
        slot.retired = grown
    slot.retired[slot.retired_count] = item
    slot.retired_count += 1

## Queue `item` behind the newest submission (the open recording, if any), or
## release it now when none is in flight.
func vulkan_retire_latest(device: VulkanDevice*, item: VulkanRetired):
    vulkan_poll(device)
    if device.submitted > device.completed or device.open != none:
        let slot = device.open else &device.slots[(usize)(device.submitted % (u64)vulkan_slot_limit)]
        vulkan_retire(slot, item) catch failure:
            _ = failure
            # Out of host memory: fall back to waiting rather than leaking or racing.
            _ = vkDeviceWaitIdle(device.device)
            vulkan_retire_through(device, device.submitted)
            vulkan_retired_free(device, item)
        return
    vulkan_retired_free(device, item)

func vulkan_retired_free(device: VulkanDevice*, item: VulkanRetired):
    if item.kind == vulkan_retired_buffer:
        vkDestroyBuffer(device.device, item.handle, none)
    elif item.kind == vulkan_retired_framebuffer:
        vkDestroyFramebuffer(device.device, item.handle, none)
    elif item.kind == vulkan_retired_descriptors:
        vkDestroyDescriptorPool(device.device, item.handle, none)
    elif item.kind == vulkan_retired_image:
        if item.view != 0: vkDestroyImageView(device.device, item.view, none)
        # A mipmapped texture's view of its first level, for frames.
        if item.extra != 0: vkDestroyImageView(device.device, item.extra, none)
        if item.handle != 0: vkDestroyImage(device.device, item.handle, none)
    elif item.kind == vulkan_retired_queries:
        vkDestroyQueryPool(device.device, item.handle, none)
    elif item.kind == vulkan_retired_accel:
        # An acceleration structure, then the buffer holding it.
        if item.handle != 0: vulkan_ray_destroy(device, item.handle)
        if item.view != 0: vkDestroyBuffer(device.device, item.view, none)
    elif item.kind == vulkan_retired_kernel:
        if item.handle != 0: vkDestroyPipeline(device.device, item.handle, none)
        if item.view != 0: vkDestroyPipelineLayout(device.device, item.view, none)
        if item.extra != 0: vkDestroyDescriptorSetLayout(device.device, item.extra, none)
    vulkan_allocation_free(device, item.allocation)

func vulkan_slot_free(device: VulkanDevice*, slot: VulkanSlot*):
    for index in 0..<slot.retired_count:
        vulkan_retired_free(device, slot.retired[index])
    slot.retired_count = 0
    slot.serial = 0

## Mark every serial up to `serial` complete and free what they held. Serials
## complete in order: a fence signalled by a submission covers every earlier
## submission on the queue.
func vulkan_retire_through(device: VulkanDevice*, serial: u64):
    while device.completed < serial:
        device.completed += 1
        vulkan_slot_free(device, &device.slots[(usize)(device.completed % (u64)vulkan_slot_limit)])

## Retire the submissions that have finished, oldest first, without waiting.
func vulkan_poll(device: VulkanDevice*):
    while device.completed < device.submitted:
        let slot = &device.slots[(usize)((device.completed + 1) % (u64)vulkan_slot_limit)]
        if vkWaitForFences(device.device, 1, &slot.fence, 1, 0) != VK_SUCCESS:
            return
        vulkan_retire_through(device, device.completed + 1)

## Wait until submission `serial` has completed, and retire it and every earlier
## one; the open recording's serial is submitted first.
func vulkan_wait(device: VulkanDevice*, serial: u64) -> !:
    if serial <= device.completed:
        return
    if serial > device.submitted:
        try vulkan_flush(device)
    let slot = &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]
    let result = vkWaitForFences(device.device, 1, &slot.fence, 1, 0xffffffffffffffffu64)
    # VK_ERROR_DEVICE_LOST: the work failed on the GPU.
    if result == -4:
        error(execution_failed, "the GPU failed while running submitted work")
    try vulkan_check(result, "Vulkan execution failed")
    vulkan_retire_through(device, serial)

## Everything submitted before a host wait on the whole queue has completed.
func vulkan_idle(device: VulkanDevice*):
    vulkan_retire_through(device, device.submitted)

## How many recordings (passes, uploads, copies) the open recording takes before
## it is submitted on its own: what bounds the latency and the draw-data ring for
## a program that renders offscreen without presenting or reading.
let vulkan_open_limit: usize = 64

## Record into the next submission: the open recording, or a new one. When the
## ring is full this waits for the oldest submission; the serial the recording
## will get is `device.submitted + 1`. A new recording starts with the
## submission's start time.
func vulkan_begin(device: VulkanDevice*) -> VulkanSlot*!:
    if let open = device.open:
        device.open_uses += 1
        return open
    let slot = try vulkan_reserve(device)
    try vulkan_check(vkResetCommandPool(device.device, slot.pool, 0), "a Vulkan command pool could not be reset")
    var begin = VkCommandBufferBeginInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT)
    try vulkan_check(vkBeginCommandBuffer(slot.command, &begin), "Vulkan recording could not begin")
    vulkan_time_start(device, slot.command)
    device.open = slot
    device.open_uses = 1
    return slot

## The ring slot the next submission takes, once free, with its command
## buffer: a frame records into it directly. That slot is the open recording's
## when there is one, so a frame submits that recording first.
func vulkan_reserve(device: VulkanDevice*) -> VulkanSlot*!:
    if device.open != none: trap("a Vulkan submission was reserved over the open recording")
    vulkan_poll(device)
    let serial = device.submitted + 1
    if serial > (u64)vulkan_slot_limit:
        try vulkan_wait(device, serial - (u64)vulkan_slot_limit)
    return &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]

## Record the next submission's start time into `command`.
func vulkan_time_start(device: VulkanDevice*, command: u64):
    if device.timing == 0: return
    let first = (u32)((device.submitted + 1) % (u64)vulkan_slot_limit) * 2
    vkCmdResetQueryPool(command, device.timing, first, 2)
    vkCmdWriteTimestamp(command, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, device.timing, first)

## Record the next submission's end time into `command`, after all its work.
func vulkan_time_end(device: VulkanDevice*, command: u64):
    if device.timing == 0: return
    let first = (u32)((device.submitted + 1) % (u64)vulkan_slot_limit) * 2
    vkCmdWriteTimestamp(command, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, device.timing, first + 1)

## Submit `submit` with `slot`'s fence as the next serial, and return it.
func vulkan_submit_reserved(device: VulkanDevice*, slot: VulkanSlot*, submit: VkSubmitInfo*) -> u64!:
    try vulkan_check(vkResetFences(device.device, 1, &slot.fence), "a Vulkan fence could not be reset")
    try vulkan_check(vkQueueSubmit(device.queue, 1, submit, slot.fence), "Vulkan submission failed")
    device.submitted += 1
    slot.serial = device.submitted
    if device.timing != 0: device.timing_serial[(usize)(device.submitted % (u64)vulkan_slot_limit)] = device.submitted
    return device.submitted

## After a failed submission: drain the device and free what the slot held.
func vulkan_abandon(device: VulkanDevice*, slot: VulkanSlot*):
    _ = vkDeviceWaitIdle(device.device)
    vulkan_retire_through(device, device.submitted)
    vulkan_slot_free(device, slot)

## End a piece of recording begun by `vulkan_begin` and return the serial its
## submission gets. The recording stays open for more, and is submitted at a
## surface's present, before a read or compute pass, when a wait asks for its
## serial, or after `vulkan_open_limit` pieces; later work is ordered after it.
func vulkan_end(device: VulkanDevice*, slot: VulkanSlot*) -> u64!:
    _ = slot
    let serial = device.submitted + 1
    if device.open_uses >= vulkan_open_limit:
        try vulkan_flush(device)
    return serial

## Submit the open recording, if any, without waiting. If submission fails, the
## device is drained and what the recording held is freed.
func vulkan_flush(device: VulkanDevice*) -> !:
    let slot = device.open else return
    device.open = none
    device.open_uses = 0
    errdefer vulkan_abandon(device, slot)
    vulkan_time_end(device, slot.command)
    try vulkan_check(vkEndCommandBuffer(slot.command), "Vulkan recording could not finish")
    var submit = VkSubmitInfo(sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, commandBufferCount = 1, pCommandBuffers = &slot.command)
    _ = try vulkan_submit_reserved(device, slot, &submit)
    vulkan_ring_trim(device)