Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.1

0.5.1 / src / gpu / vulkan / device.lucb

#==============================================================================================
#
#   device - Vulkan instance, device and queues
#
#   DESCRIPTION:
#       Vulkan 1.0: VK_MAKE_API_VERSION(0, 1, 0, 0).
#
#==============================================================================================

let vulkan_api_version_1_0: u32 = 1u32 << 22

## One render pass pair and pipeline pair per color format in use.
struct VulkanFormatState:
    var format: u32
    var pass_present: u64
    var pass_texture: u64
    var overlay: u64
    var depth_test: u64
    var geometry: u64
    # Mesh pipelines by MeshKind (vulkan/mesh.lucb).
    var mesh: u64[3]

## A free range of a memory page.
struct VulkanRange:
    var offset: u64
    var size: u64

## One large VkDeviceMemory block that images are carved from, first fit. Its
## free ranges are sorted by offset and never adjacent.
struct VulkanPage:
    var memory: u64
    var size: u64
    var type_index: u32
    var live: usize
    var ranges: VulkanRange[]
    var range_count: usize
    var next: VulkanPage*?

## Where a resource's memory lives: a range of a page, or, with no page, a
## dedicated allocation of its own.
struct VulkanAllocation:
    var memory: u64
    var offset: u64
    var size: u64
    var page: VulkanPage*?

let vulkan_retired_buffer: u8 = 1
let vulkan_retired_framebuffer: u8 = 2
let vulkan_retired_descriptors: u8 = 3
let vulkan_retired_image: u8 = 4

## A handle released once the submission holding it has completed.
struct VulkanRetired:
    var kind: u8
    var handle: u64
    var view: u64
    var allocation: VulkanAllocation

## One in-flight submission: its own command pool and buffer, the fence it
## signals, and what it frees when that fence is signalled. `serial` is zero
## while the slot is idle or recording.
struct VulkanSlot:
    var pool: u64
    var command: u64
    var fence: u64
    var serial: u64
    var retired: VulkanRetired[]
    var retired_count: usize

## How many texture submissions may be in flight before recording one waits
## for the oldest.
let vulkan_slot_limit: usize = 32

struct VulkanDevice:
    var instance: u64
    var physical: u64
    var device: u64
    var queue: u64
    var family: u32
    var memory: VkPhysicalDeviceMemoryProperties
    # Drawing state shared by surfaces and textures; see vulkan_drawing.
    var ready: bool
    # The submission ring: serial n uses slots[n % vulkan_slot_limit]. Every
    # serial up to `completed` has finished; `submitted` is the newest.
    var slots: VulkanSlot[32]
    var submitted: u64
    var completed: u64
    # Image memory pages; see vulkan_image_memory.
    var pages: VulkanPage*?
    var page_count: usize
    # Device-local bytes held for images: pages and dedicated image memory.
    var image_bytes: u64
    var integrated: bool
    var descriptor_layout: u64
    var layout: u64
    var nearest: u64
    var linear: u64
    var placeholder: VulkanTexture*?
    var fill: u64
    var formats: VulkanFormatState[6]
    var format_count: usize
    var depth_image: u64
    var depth_memory: u64
    var depth_view: u64
    var depth_width: u32
    var depth_height: u32
    # The staging rings; see vulkan/transfer.lucb.
    var reads: VulkanRingSlot[3]
    var uploads: VulkanRingSlot[4]
    var upload_slot: usize
    # Mesh draws: their descriptor and pipeline layouts and a buffer bound
    # where a draw leaves one out; see vulkan/mesh.lucb.
    var mesh_layout: u64
    var mesh_pipeline_layout: u64
    var mesh_placeholder: VulkanBuffer*?

func vulkan_check(result: i32, message: str) -> !:
    # VK_ERROR_OUT_OF_HOST_MEMORY and VK_ERROR_OUT_OF_DEVICE_MEMORY: room can be made.
    if result == -1 or result == -2:
        error(out_of_memory, message)
    if result != VK_SUCCESS:
        error(failed, message)

func vulkan_device_open() -> VulkanDevice*!:
    try vulkan_load()
    let state = try new VulkanDevice() in memory.heap
    errdefer vulkan_device_destroy(state)
    var application = VkApplicationInfo(sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
                                        pApplicationName = "Luce", pEngineName = "Luce", apiVersion = vulkan_api_version_1_0)
    let extensions = try vulkan_presentation_extensions()
    var instance_info = VkInstanceCreateInfo(sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
                                             pApplicationInfo = &application, enabledExtensionCount = (u32)extensions.length, ppEnabledExtensionNames = (c.str*)extensions.data)
    if vkCreateInstance(&instance_info, none, &state.instance) != 0:
        error(unavailable, "a Vulkan instance with native presentation is unavailable")
    var count: u32 = 0
    try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, none), "Vulkan devices could not be enumerated")
    if count == 0:
        error(unavailable, "no Vulkan device is available")
    let devices = try new u64[(usize)count] ---
    defer free(devices)
    try vulkan_check(vkEnumeratePhysicalDevices(state.instance, &count, devices.data), "Vulkan devices changed during enumeration")
    # a discrete GPU first, then an integrated one, then anything else: on a laptop with
    # both, drawing belongs on the one built for it
    let order = try new u64[(usize)count] ---
    defer free(order)
    var ordered: usize
    for rank in 0..<3:
        for physical in devices[..<(usize)count]:
            if vulkan_device_rank(physical) == (u32)rank:
                order[ordered] = physical
                ordered += 1
    for index in 0..<ordered:
        let physical = order[index]
        var families: u32 = 0
        vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, none)
        let properties = try new VkQueueFamilyProperties[(usize)families] ---
        defer free(properties)
        vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, properties.data)
        for family in 0..<(usize)families:
            if properties[family].queueCount == 0 or (properties[family].queueFlags & VK_QUEUE_GRAPHICS_BIT) == 0 or not vulkan_presentation_support(physical, (u32)family):
                continue
            var priority: f32 = 1.0
            var queue_info = VkDeviceQueueCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
                                                     queueFamilyIndex = (u32)family, queueCount = 1, pQueuePriorities = &priority)
            var device_extension: c.str = VK_KHR_SWAPCHAIN_EXTENSION_NAME
            var device_info = VkDeviceCreateInfo(sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
                                                 queueCreateInfoCount = 1, pQueueCreateInfos = &queue_info,
                                                 enabledExtensionCount = 1, ppEnabledExtensionNames = &device_extension)
            if vkCreateDevice(physical, &device_info, none, &state.device) != 0:
                continue
            state.physical = physical
            state.integrated = vulkan_device_rank(physical) == 1
            state.family = (u32)family
            vkGetDeviceQueue(state.device, state.family, 0, &state.queue)
            vkGetPhysicalDeviceMemoryProperties(physical, &state.memory)
            return state
    error(unavailable, "no Vulkan graphics queue supports native presentation")

## 0 for a discrete GPU, 1 for an integrated one, 2 for any other kind. Only
## VkPhysicalDeviceProperties' device type is read (bytes 16..20); the bindings
## leave the struct out, and it is under 1 KiB on every platform.
func vulkan_device_rank(physical: u64) -> u32:
    var properties: u8[1024] = ---
    vkGetPhysicalDeviceProperties(physical, (void*)&properties[0])
    let kind = (u32)properties[16] | ((u32)properties[17] << 8) | ((u32)properties[18] << 16) | ((u32)properties[19] << 24)
    if kind == 2:
        return 0
    if kind == 1:
        return 1
    return 2

## The budget: a discrete GPU's largest device-local heap; an integrated
## GPU's largest heap of any kind, since its textures live in system memory
## (an APU's device-local heap is often a small carve-out, 512 MB, while its
## images spill into the far larger host heap). The image memory held as used.
func vulkan_memory_report(device: VulkanDevice*) -> DeviceMemory:
    vulkan_poll(device)
    var budget: u64 = 0
    for index in 0..<(usize)device.memory.memoryHeapCount:
        let heap = device.memory.memoryHeaps[index]
        # VK_MEMORY_HEAP_DEVICE_LOCAL_BIT
        if (device.integrated or (heap.flags & 1) != 0) and heap.size > budget:
            budget = heap.size
    return DeviceMemory(budget = budget, used = device.image_bytes, unified = device.integrated)

## Retire finished submissions and count the image memory pages held.
func vulkan_memory_pages(device: VulkanDevice*) -> usize:
    vulkan_poll(device)
    return device.page_count

func vulkan_device_destroy(state: VulkanDevice*):
    if state.device != 0:
        _ = vkDeviceWaitIdle(state.device)
        vulkan_drawing_destroy(state)
        # Textures still open when the device goes release their pages with it.
        while let page = state.pages:
            vulkan_page_destroy(state, page)
        vkDestroyDevice(state.device, none)
    if state.instance != 0:
        vkDestroyInstance(state.instance, none)
    free(state) in memory.heap

func vulkan_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u32!:
    for index in 0..<(usize)device.memory.memoryTypeCount:
        if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
            return (u32)index
    error(unavailable, "no compatible Vulkan memory type is available")

## Whether a memory type has every property in `flags`.
func vulkan_has_memory_type(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> bool:
    for index in 0..<(usize)device.memory.memoryTypeCount:
        if (requirements.memoryTypeBits & (1u32 << (u32)index)) != 0 and (device.memory.memoryTypes[index].propertyFlags & flags) == flags:
            return true
    return false

func vulkan_allocate(device: VulkanDevice*, size: u64, type_index: u32) -> u64!:
    var info = VkMemoryAllocateInfo(sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, allocationSize = size, memoryTypeIndex = type_index)
    var block: u64 = 0
    try vulkan_check(vkAllocateMemory(device.device, &info, none, &block), "Vulkan device memory allocation failed")
    return block

## A dedicated allocation: staging, vertex and depth memory.
func vulkan_memory(device: VulkanDevice*, requirements: VkMemoryRequirements, flags: u32) -> u64!:
    return try vulkan_allocate(device, requirements.size, try vulkan_memory_type(device, requirements, flags))

# mark: Image memory pages ====================================================================

## Pages are 64 MiB, or 256 MiB when their heap holds more than 8 GiB.
func vulkan_page_size(device: VulkanDevice*, type_index: u32) -> u64:
    let heap = device.memory.memoryHeaps[(usize)device.memory.memoryTypes[(usize)type_index].heapIndex]
    if heap.size > (8u64 << 30):
        return 256u64 << 20
    return 64u64 << 20

## Device-local memory for an image. Images up to a quarter of a page share
## pages, so a large document makes a few allocations rather than one per tile
## (Windows drivers allow about 4096); bigger ones get memory of their own.
func vulkan_image_memory(device: VulkanDevice*, requirements: VkMemoryRequirements) -> VulkanAllocation!:
    let type_index = try vulkan_memory_type(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
    let page_size = vulkan_page_size(device, type_index)
    if requirements.size > page_size // 4:
        let dedicated = VulkanAllocation(memory = try vulkan_allocate(device, requirements.size, type_index), size = requirements.size)
        device.image_bytes += requirements.size
        return dedicated
    var cursor = device.pages
    while let page = cursor:
        if page.type_index == type_index:
            if let offset = try vulkan_page_take(page, requirements.size, requirements.alignment):
                return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)
        cursor = page.next
    let page = try vulkan_page_create(device, type_index, page_size)
    let offset = (try vulkan_page_take(page, requirements.size, requirements.alignment)) else trap("a fresh Vulkan memory page cannot hold a quarter of itself")
    return VulkanAllocation(memory = page.memory, offset = offset, size = requirements.size, page = page)

func vulkan_page_create(device: VulkanDevice*, type_index: u32, size: u64) -> VulkanPage*!:
    let memory_block = try vulkan_allocate(device, size, type_index)
    errdefer vkFreeMemory(device.device, memory_block, none)
    let page = try new VulkanPage(memory = memory_block, size = size, type_index = type_index) in memory.heap
    errdefer free(page) in memory.heap
    page.ranges = try new VulkanRange[8] --- in memory.heap
    page.ranges[0] = VulkanRange(offset = 0, size = size)
    page.range_count = 1
    page.next = device.pages
    device.pages = page
    device.page_count += 1
    device.image_bytes += size
    return page

func vulkan_page_destroy(device: VulkanDevice*, page: VulkanPage*):
    if device.pages == page:
        device.pages = page.next
    else:
        var cursor = device.pages
        while let previous = cursor:
            if previous.next == page:
                previous.next = page.next
                break
            cursor = previous.next
    vkFreeMemory(device.device, page.memory, none)
    let size = page.size
    free(page.ranges) in memory.heap
    free(page) in memory.heap
    device.page_count -= 1
    device.image_bytes -= size if size <= device.image_bytes else device.image_bytes

## First fit: the aligned offset of `size` bytes taken from a free range, or none
## when no range fits. Free ranges never outnumber live allocations plus one, so
## growing the list to that before taking leaves room for every later release.
func vulkan_page_take(page: VulkanPage*, size: u64, alignment: u64) -> u64?!:
    if page.ranges.length < page.live + 2:
        let grown = try new VulkanRange[(page.live + 2) * 2] --- in memory.heap
        memory.copy(grown, page.ranges, page.range_count)
        free(page.ranges) in memory.heap
        page.ranges = grown
    let align = alignment if alignment > 0 else 1u64
    for index in 0..<page.range_count:
        let range = page.ranges[index]
        let start = (range.offset + align - 1) // align * align
        let end = range.offset + range.size
        if start + size > end:
            continue
        let prefix = start - range.offset
        let suffix = end - (start + size)
        if prefix == 0 and suffix == 0:
            for later in index..<(page.range_count - 1):
                page.ranges[later] = page.ranges[later + 1]
            page.range_count -= 1
        elif prefix == 0:
            page.ranges[index] = VulkanRange(offset = start + size, size = suffix)
        elif suffix == 0:
            page.ranges[index].size = prefix
        else:
            var later = page.range_count
            while later > index + 1:
                page.ranges[later] = page.ranges[later - 1]
                later -= 1
            page.ranges[index].size = prefix
            page.ranges[index + 1] = VulkanRange(offset = start + size, size = suffix)
            page.range_count += 1
        page.live += 1
        return start
    return none

## Return a range to its page, merging it with the free ranges beside it.
func vulkan_page_give(page: VulkanPage*, offset: u64, size: u64):
    var index: usize = 0
    while index < page.range_count and page.ranges[index].offset < offset:
        index += 1
    let joins_previous = index > 0 and page.ranges[index - 1].offset + page.ranges[index - 1].size == offset
    let joins_next = index < page.range_count and offset + size == page.ranges[index].offset
    if joins_previous and joins_next:
        page.ranges[index - 1].size += size + page.ranges[index].size
        for later in index..<(page.range_count - 1):
            page.ranges[later] = page.ranges[later + 1]
        page.range_count -= 1
    elif joins_previous:
        page.ranges[index - 1].size += size
    elif joins_next:
        page.ranges[index] = VulkanRange(offset = offset, size = size + page.ranges[index].size)
    else:
        var later = page.range_count
        while later > index:
            page.ranges[later] = page.ranges[later - 1]
            later -= 1
        page.ranges[index] = VulkanRange(offset = offset, size = size)
        page.range_count += 1
    page.live -= 1

## Release memory from `vulkan_image_memory` or a dedicated allocation; a page
## is freed as soon as nothing lives in it.
func vulkan_allocation_free(device: VulkanDevice*, allocation: VulkanAllocation):
    if let page = allocation.page:
        vulkan_page_give(page, allocation.offset, allocation.size)
        if page.live == 0:
            vulkan_page_destroy(device, page)
    elif allocation.memory != 0:
        vkFreeMemory(device.device, allocation.memory, none)
        # Dedicated image memory carries its size; staging buffers carry none.
        device.image_bytes -= allocation.size if allocation.size <= device.image_bytes else device.image_bytes

# mark: Submissions ===========================================================================

func vulkan_slots_create(device: VulkanDevice*) -> !:
    for index in 0..<vulkan_slot_limit:
        let slot = &device.slots[index]
        var pool_info = VkCommandPoolCreateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, queueFamilyIndex = device.family)
        try vulkan_check(vkCreateCommandPool(device.device, &pool_info, none, &slot.pool), "a Vulkan command pool could not be created")
        var command_info = VkCommandBufferAllocateInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, commandPool = slot.pool, commandBufferCount = 1)
        try vulkan_check(vkAllocateCommandBuffers(device.device, &command_info, &slot.command), "a Vulkan command buffer could not be allocated")
        var fence_info = VkFenceCreateInfo(sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO)
        try vulkan_check(vkCreateFence(device.device, &fence_info, none, &slot.fence), "a Vulkan fence could not be created")
        slot.retired = try new VulkanRetired[8] --- in memory.heap

## Wait for the device, free everything held, and destroy the ring.
func vulkan_slots_destroy(device: VulkanDevice*):
    _ = vkDeviceWaitIdle(device.device)
    vulkan_retire_through(device, device.submitted)
    for index in 0..<vulkan_slot_limit:
        let slot = &device.slots[index]
        vulkan_slot_free(device, slot)
        if slot.fence != 0: vkDestroyFence(device.device, slot.fence, none)
        if slot.pool != 0: vkDestroyCommandPool(device.device, slot.pool, none)
        if slot.retired.length > 0: free(slot.retired) in memory.heap
        *slot = VulkanSlot()

## Queue `item` for release when `slot`'s submission completes.
func vulkan_retire(slot: VulkanSlot*, item: VulkanRetired) -> !:
    if slot.retired_count == slot.retired.length:
        let grown = try new VulkanRetired[slot.retired.length * 2 + 8] --- in memory.heap
        memory.copy(grown, slot.retired, slot.retired_count)
        if slot.retired.length > 0: free(slot.retired) in memory.heap
        slot.retired = grown
    slot.retired[slot.retired_count] = item
    slot.retired_count += 1

## Queue `item` behind the newest submission, or release it now when none is in flight.
func vulkan_retire_latest(device: VulkanDevice*, item: VulkanRetired):
    vulkan_poll(device)
    if device.submitted > device.completed:
        let slot = &device.slots[(usize)(device.submitted % (u64)vulkan_slot_limit)]
        vulkan_retire(slot, item) catch failure:
            _ = failure
            # Out of host memory: fall back to waiting rather than leaking or racing.
            _ = vkDeviceWaitIdle(device.device)
            vulkan_retire_through(device, device.submitted)
            vulkan_retired_free(device, item)
        return
    vulkan_retired_free(device, item)

func vulkan_retired_free(device: VulkanDevice*, item: VulkanRetired):
    if item.kind == vulkan_retired_buffer:
        vkDestroyBuffer(device.device, item.handle, none)
    elif item.kind == vulkan_retired_framebuffer:
        vkDestroyFramebuffer(device.device, item.handle, none)
    elif item.kind == vulkan_retired_descriptors:
        vkDestroyDescriptorPool(device.device, item.handle, none)
    elif item.kind == vulkan_retired_image:
        if item.view != 0: vkDestroyImageView(device.device, item.view, none)
        if item.handle != 0: vkDestroyImage(device.device, item.handle, none)
    vulkan_allocation_free(device, item.allocation)

func vulkan_slot_free(device: VulkanDevice*, slot: VulkanSlot*):
    for index in 0..<slot.retired_count:
        vulkan_retired_free(device, slot.retired[index])
    slot.retired_count = 0
    slot.serial = 0

## Mark every serial up to `serial` complete and free what they held. Serials
## complete in order: a fence signalled by a submission covers every earlier
## submission on the queue.
func vulkan_retire_through(device: VulkanDevice*, serial: u64):
    while device.completed < serial:
        device.completed += 1
        vulkan_slot_free(device, &device.slots[(usize)(device.completed % (u64)vulkan_slot_limit)])

## Retire the submissions that have finished, oldest first, without waiting.
func vulkan_poll(device: VulkanDevice*):
    while device.completed < device.submitted:
        let slot = &device.slots[(usize)((device.completed + 1) % (u64)vulkan_slot_limit)]
        if vkWaitForFences(device.device, 1, &slot.fence, 1, 0) != VK_SUCCESS:
            return
        vulkan_retire_through(device, device.completed + 1)

## Wait until submission `serial` has completed, and retire it and every earlier one.
func vulkan_wait(device: VulkanDevice*, serial: u64) -> !:
    if serial <= device.completed:
        return
    let slot = &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]
    try vulkan_check(vkWaitForFences(device.device, 1, &slot.fence, 1, 0xffffffffffffffffu64), "Vulkan execution failed")
    vulkan_retire_through(device, serial)

## Everything submitted before a host wait on the whole queue has completed.
func vulkan_idle(device: VulkanDevice*):
    vulkan_retire_through(device, device.submitted)

## Begin recording the next submission. When the ring is full this waits for the
## oldest submission; the serial the recording will get is `device.submitted + 1`.
func vulkan_begin(device: VulkanDevice*) -> VulkanSlot*!:
    vulkan_poll(device)
    let serial = device.submitted + 1
    if serial > (u64)vulkan_slot_limit:
        try vulkan_wait(device, serial - (u64)vulkan_slot_limit)
    let slot = &device.slots[(usize)(serial % (u64)vulkan_slot_limit)]
    try vulkan_check(vkResetCommandPool(device.device, slot.pool, 0), "a Vulkan command pool could not be reset")
    var begin = VkCommandBufferBeginInfo(sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT)
    try vulkan_check(vkBeginCommandBuffer(slot.command, &begin), "Vulkan recording could not begin")
    return slot

## After a failed submission: drain the device and free what the slot held.
func vulkan_abandon(device: VulkanDevice*, slot: VulkanSlot*):
    _ = vkDeviceWaitIdle(device.device)
    vulkan_retire_through(device, device.submitted)
    vulkan_slot_free(device, slot)

## Submit the recording begun by `vulkan_begin` without waiting, and return its
## serial. Later submissions on the queue are ordered after it. If submission
## fails, the device is drained and what the slot held is freed.
func vulkan_end(device: VulkanDevice*, slot: VulkanSlot*) -> u64!:
    errdefer vulkan_abandon(device, slot)
    try vulkan_check(vkEndCommandBuffer(slot.command), "Vulkan recording could not finish")
    try vulkan_check(vkResetFences(device.device, 1, &slot.fence), "a Vulkan fence could not be reset")
    var submit = VkSubmitInfo(sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, commandBufferCount = 1, pCommandBuffers = &slot.command)
    try vulkan_check(vkQueueSubmit(device.queue, 1, &submit, slot.fence), "Vulkan submission failed")
    device.submitted += 1
    slot.serial = device.submitted
    return device.submitted