Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-gpu

Source of release 0.5.4

0.5.4 / src / gpu / vulkan / texture.lucb

#==============================================================================================
#
#   texture - Vulkan images, samplers, transfers and offscreen passes
#
#   DESCRIPTION:
#       Device-local Vulkan images with tracked layouts, carved from shared memory pages.
#       Uploads and offscreen passes are submitted on the device's submission ring without
#       waiting: the queue orders them before every later submission, and what they used is
#       freed when their fence signals. Only a readback waits. An image is destroyed once the
#       last submission using it has completed. Shared drawing state (layouts, samplers, the
#       placeholder image, the depth attachment) lives on the device and is built on first use.
#
#==============================================================================================

struct VulkanTexture:
    var image: u64
    var memory: VulkanAllocation
    var view: u64
    # The newest submission that renders into, uploads to, reads or samples the image.
    var last_use: u64
    var width: u32
    var height: u32
    var format: Format
    var native_format: u32
    var layout: u32
    ## Mip levels; with more than one, `attachment` views the first for frames.
    var levels: u32 = 1
    var attachment: u64
    ## Whether the device filters the format linearly; sampling falls back to
    ## nearest where it does not (32-bit floats on some devices).
    var filterable: bool

func vulkan_native_format(pixel_format: Format) -> u32:
    if pixel_format == Format.rgba8_linear: return VK_FORMAT_R8G8B8A8_UNORM
    if pixel_format == Format.rgba16_float: return VK_FORMAT_R16G16B16A16_SFLOAT
    if pixel_format == Format.r8: return VK_FORMAT_R8_UNORM
    if pixel_format == Format.rgba32_float: return VK_FORMAT_R32G32B32A32_SFLOAT
    if pixel_format == Format.r32_float: return VK_FORMAT_R32_SFLOAT
    if pixel_format == Format.r32_uint: return VK_FORMAT_R32_UINT
    if pixel_format == Format.rg32_float: return VK_FORMAT_R32G32_SFLOAT
    return VK_FORMAT_R8G8B8A8_SRGB

# mark: Shared drawing state ==================================================================

## Layouts, samplers, the placeholder image and the submission ring,
## created on first use by any surface or texture.
func vulkan_drawing(device: VulkanDevice*) -> !:
    if device.ready:
        return
    errdefer vulkan_drawing_destroy(device)
    try vulkan_slots_create(device)
    # The fragment contract: sampled images at bindings 1..4, coverage words at 5,
    # uniforms as push constants. Vertex stages pull from 6 and 7 (retained
    # geometry and its parameters; shade_quads' records and parameters) and 8
    # (shade_quads' order).
    var bindings: VkDescriptorSetLayoutBinding[8]
    for index in 0..<(usize)4:
        bindings[index] = VkDescriptorSetLayoutBinding(binding = (u32)index + 1, descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, descriptorCount = 1, stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT)
    bindings[4] = VkDescriptorSetLayoutBinding(binding = 5, descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = 1, stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT)
    bindings[5] = VkDescriptorSetLayoutBinding(binding = 6, descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = 1, stageFlags = VK_SHADER_STAGE_VERTEX_BIT)
    bindings[6] = VkDescriptorSetLayoutBinding(binding = 7, descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = 1, stageFlags = VK_SHADER_STAGE_VERTEX_BIT)
    bindings[7] = VkDescriptorSetLayoutBinding(binding = 8, descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, descriptorCount = 1, stageFlags = VK_SHADER_STAGE_VERTEX_BIT)
    var descriptor_info = VkDescriptorSetLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, bindingCount = 8, pBindings = &bindings[0])
    try vulkan_check(vkCreateDescriptorSetLayout(device.device, &descriptor_info, none, &device.descriptor_layout), "the Vulkan draw layout could not be created")
    var push = VkPushConstantRange(stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT, size = (u32)uniform_limit)
    var layout_info = VkPipelineLayoutCreateInfo(sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
                                                 setLayoutCount = 1, pSetLayouts = &device.descriptor_layout, pushConstantRangeCount = 1, pPushConstantRanges = &push)
    try vulkan_check(vkCreatePipelineLayout(device.device, &layout_info, none, &device.layout), "the Vulkan pipeline layout could not be created")
    device.nearest = try vulkan_sampler(device, VK_FILTER_NEAREST)
    device.linear = try vulkan_sampler(device, VK_FILTER_LINEAR)
    device.trilinear = try vulkan_sampler(device, VK_FILTER_LINEAR, true)
    device.fill = try vulkan_shader(device, fill_frag_words)
    device.ready = true
    # Every draw binds an image; one opaque texel stands in for draws that sample none.
    let placeholder = try vulkan_texture_create(device, 1, 1, Format.rgba8_linear, false)
    device.placeholder = placeholder
    var white: u8[4] = [255, 255, 255, 255]
    try vulkan_texture_upload(device, placeholder, white, Region(0, 0, 1, 1))

## A clamped sampler; `mipmapped` also blends linearly between levels, where
## the others read the first only (a maximum level of detail of zero).
func vulkan_sampler(device: VulkanDevice*, filter: u32, mipmapped: bool = false) -> u64!:
    var info = VkSamplerCreateInfo(sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, magFilter = filter, minFilter = filter,
                                   mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR if mipmapped else VK_SAMPLER_MIPMAP_MODE_NEAREST,
                                   addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
                                   maxAnisotropy = 1.0, maxLod = 1000.0 if mipmapped else 0.0)
    var sampler: u64 = 0
    try vulkan_check(vkCreateSampler(device.device, &info, none, &sampler), "a Vulkan sampler could not be created")
    return sampler

func vulkan_drawing_destroy(device: VulkanDevice*):
    # What is still recorded runs before the device goes.
    vulkan_flush(device) catch failure:
        _ = failure
    _ = vkDeviceWaitIdle(device.device)
    vulkan_idle(device)
    vulkan_mesh_destroy(device)
    if let placeholder = device.placeholder:
        device.placeholder = none
        vulkan_texture_destroy(device, placeholder)
    vulkan_depth_destroy(device)
    vulkan_depth_release(device, &device.depth_copy_image, &device.depth_copy_memory, &device.depth_copy_view)
    device.depth_copy_width = 0
    device.depth_copy_height = 0
    vulkan_rings_destroy(device)
    vulkan_slots_destroy(device)
    vulkan_ring_destroy(device)
    for state in device.formats[..<device.format_count]:
        if state.overlay != 0: vkDestroyPipeline(device.device, state.overlay, none)
        if state.depth_test != 0: vkDestroyPipeline(device.device, state.depth_test, none)
        if state.geometry != 0: vkDestroyPipeline(device.device, state.geometry, none)
        for pipeline in state.mesh:
            if pipeline != 0: vkDestroyPipeline(device.device, pipeline, none)
        if state.pass_present != 0: vkDestroyRenderPass(device.device, state.pass_present, none)
        if state.pass_texture != 0: vkDestroyRenderPass(device.device, state.pass_texture, none)
        if state.pass_load != 0: vkDestroyRenderPass(device.device, state.pass_load, none)
        vulkan_split_passes_destroy(device, state.split)
    device.format_count = 0
    if device.nearest != 0: vkDestroySampler(device.device, device.nearest, none)
    if device.linear != 0: vkDestroySampler(device.device, device.linear, none)
    if device.trilinear != 0: vkDestroySampler(device.device, device.trilinear, none)
    device.trilinear = 0
    if device.fill != 0: vkDestroyShaderModule(device.device, device.fill, none)
    device.fill = 0
    if device.layout != 0: vkDestroyPipelineLayout(device.device, device.layout, none)
    if device.descriptor_layout != 0: vkDestroyDescriptorSetLayout(device.device, device.descriptor_layout, none)
    device.nearest = 0
    device.linear = 0
    device.layout = 0
    device.descriptor_layout = 0
    device.ready = false

## Allocate a 32-bit depth image of `width` by `height`: the image, its
## memory and its view are written to `depth`, `block` and `target`, replacing
## whatever they held. An attachment is also a copy source (scene depth
## copies); with `sampled` it is instead a copy target draws sample.
func vulkan_depth_create(device: VulkanDevice*, width: u32, height: u32, depth: u64*, block: u64*, target: u64*, sampled: bool = false) -> !:
    vulkan_depth_release(device, depth, block, target)
    errdefer vulkan_depth_release(device, depth, block, target)
    var properties: VkFormatProperties
    vkGetPhysicalDeviceFormatProperties(device.physical, VK_FORMAT_D32_SFLOAT, &properties)
    if (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) == 0:
        error(unavailable, "the Vulkan device has no 32-bit depth attachment support")
    if sampled and (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) == 0:
        error(unsupported, "the Vulkan device cannot sample 32-bit depth, so draws cannot read the scene depth")
    let usage = (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT) if sampled else (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT)
    var info = VkImageCreateInfo(sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, imageType = VK_IMAGE_TYPE_2D, format = VK_FORMAT_D32_SFLOAT,
                                 extent = VkExtent3D(width = width, height = height, depth = 1), mipLevels = 1, arrayLayers = 1, samples = VK_SAMPLE_COUNT_1_BIT, usage = usage)
    try vulkan_check(vkCreateImage(device.device, &info, none, depth), "the Vulkan depth image could not be created")
    var requirements: VkMemoryRequirements
    vkGetImageMemoryRequirements(device.device, *depth, &requirements)
    *block = try vulkan_memory(device, requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
    try vulkan_check(vkBindImageMemory(device.device, *depth, *block, 0), "Vulkan depth memory could not be bound")
    *target = try vulkan_image_view(device, *depth, VK_FORMAT_D32_SFLOAT, VK_IMAGE_ASPECT_DEPTH_BIT)

## Free a depth attachment made by `vulkan_depth_create`, and zero its handles.
func vulkan_depth_release(device: VulkanDevice*, depth: u64*, block: u64*, target: u64*):
    if *target != 0: vkDestroyImageView(device.device, *target, none)
    if *depth != 0: vkDestroyImage(device.device, *depth, none)
    if *block != 0: vkFreeMemory(device.device, *block, none)
    *target = 0
    *depth = 0
    *block = 0

## The device's depth attachment for texture passes. It only grows: a
## framebuffer may be smaller than its attachments, so passes of every size
## share it, and tiles, thumbnails and glyphs drawn in turn never make it
## again. Only a texture pass's own framebuffer may name it, and that
## framebuffer is retired with its submission; a replaced attachment is retired
## behind the newest submission, which is why the swapchain keeps a depth
## attachment of its own.
func vulkan_depth_for(device: VulkanDevice*, width: u32, height: u32) -> u64!:
    if device.depth_view != 0 and width <= device.depth_width and height <= device.depth_height:
        return device.depth_view
    let grown_width = width if width > device.depth_width else device.depth_width
    let grown_height = height if height > device.depth_height else device.depth_height
    device.depth_width = 0
    device.depth_height = 0
    if device.depth_image != 0:
        vulkan_retire_latest(device, VulkanRetired(kind = vulkan_retired_image, handle = device.depth_image, view = device.depth_view,
                                                   allocation = VulkanAllocation(memory = device.depth_memory)))
        device.depth_image = 0
        device.depth_view = 0
        device.depth_memory = 0
    try vulkan_depth_create(device, grown_width, grown_height, &device.depth_image, &device.depth_memory, &device.depth_view)
    device.depth_width = grown_width
    device.depth_height = grown_height
    return device.depth_view

## The image draws reading the scene depth sample, at least `width` by
## `height`: grown and retired as `vulkan_depth_for` grows the attachment.
func vulkan_depth_copy_for(device: VulkanDevice*, width: u32, height: u32) -> !:
    if device.depth_copy_view != 0 and width <= device.depth_copy_width and height <= device.depth_copy_height:
        return
    let grown_width = width if width > device.depth_copy_width else device.depth_copy_width
    let grown_height = height if height > device.depth_copy_height else device.depth_copy_height
    device.depth_copy_width = 0
    device.depth_copy_height = 0
    if device.depth_copy_image != 0:
        vulkan_retire_latest(device, VulkanRetired(kind = vulkan_retired_image, handle = device.depth_copy_image, view = device.depth_copy_view,
                                                   allocation = VulkanAllocation(memory = device.depth_copy_memory)))
        device.depth_copy_image = 0
        device.depth_copy_view = 0
        device.depth_copy_memory = 0
    try vulkan_depth_create(device, grown_width, grown_height, &device.depth_copy_image, &device.depth_copy_memory, &device.depth_copy_view, sampled = true)
    device.depth_copy_width = grown_width
    device.depth_copy_height = grown_height
    device.depth_copy_layout = VK_IMAGE_LAYOUT_UNDEFINED

func vulkan_depth_destroy(device: VulkanDevice*):
    vulkan_depth_release(device, &device.depth_image, &device.depth_memory, &device.depth_view)
    device.depth_width = 0
    device.depth_height = 0

# mark: Images ================================================================================

func vulkan_texture_create(device: VulkanDevice*, width: u32, height: u32, pixel_format: Format, storage: bool, levels: u32 = 1) -> VulkanTexture*!:
    try vulkan_drawing(device)
    let native_format = vulkan_native_format(pixel_format)
    var properties: VkFormatProperties
    vkGetPhysicalDeviceFormatProperties(device.physical, native_format, &properties)
    let features = properties.optimalTilingFeatures
    if (features & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) == 0 or (storage and (features & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) == 0):
        error(unsupported, "the Vulkan device cannot sample or store textures of this format")
    let blits = VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_BLIT_DST_BIT | VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT
    if levels > 1 and (features & blits) != blits:
        error(unsupported, "the Vulkan device cannot make mipmaps of this format")
    var usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT
    if renderable(pixel_format): usage = usage | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
    if storage: usage = usage | VK_IMAGE_USAGE_STORAGE_BIT
    let state = try new VulkanTexture(width = width, height = height, format = pixel_format, native_format = native_format, layout = VK_IMAGE_LAYOUT_UNDEFINED, levels = levels,
                                      filterable = (features & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT) != 0) in memory.heap
    errdefer vulkan_texture_destroy(device, state)
    var info = VkImageCreateInfo(sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, imageType = VK_IMAGE_TYPE_2D, format = state.native_format,
                                 extent = VkExtent3D(width = width, height = height, depth = 1), mipLevels = levels, arrayLayers = 1, samples = VK_SAMPLE_COUNT_1_BIT,
                                 usage = usage)
    try vulkan_check(vkCreateImage(device.device, &info, none, &state.image), "a Vulkan image could not be created")
    var requirements: VkMemoryRequirements
    vkGetImageMemoryRequirements(device.device, state.image, &requirements)
    state.memory = try vulkan_image_memory(device, requirements)
    try vulkan_check(vkBindImageMemory(device.device, state.image, state.memory.memory, state.memory.offset), "Vulkan image memory could not be bound")
    state.view = try vulkan_image_view(device, state.image, state.native_format, VK_IMAGE_ASPECT_COLOR_BIT, levels)
    # A framebuffer names one level: the first.
    if levels > 1:
        state.attachment = try vulkan_image_view(device, state.image, state.native_format, VK_IMAGE_ASPECT_COLOR_BIT)
    # The layout stays undefined until the first use: a pass rendering into it
    # starts from undefined anyway, and one sampling it first makes it ready
    # in its own commands (`vulkan_encode`), so creating costs no submission.
    return state

## Destroy the image now if every submission using it has completed, or else
## when the last of them does.
func vulkan_texture_destroy(device: VulkanDevice*, state: VulkanTexture*):
    let item = VulkanRetired(kind = vulkan_retired_image, handle = state.image, view = state.view, extra = state.attachment, allocation = state.memory)
    let last_use = state.last_use
    free(state) in memory.heap
    vulkan_poll(device)
    if last_use <= device.completed:
        vulkan_retired_free(device, item)
        return
    vulkan_retire(&device.slots[(usize)(last_use % (u64)vulkan_slot_limit)], item) catch failure:
        _ = failure
        # Out of host memory: wait rather than leak or free an image in use.
        vulkan_wait(device, last_use) catch waited:
            _ = waited
        vulkan_retired_free(device, item)

## Record a layout transition of the whole image, with the broadest matching
## access masks. Its first scope covers earlier submissions on the queue too, so
## an image written by work still in flight is safe to use in the next one.
func vulkan_transition(command: u64, state: VulkanTexture*, layout: u32):
    if state.layout == layout:
        return
    var barrier = VkImageMemoryBarrier(sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
                                       srcAccessMask = vulkan_image_access, dstAccessMask = vulkan_image_access,
                                       oldLayout = state.layout, newLayout = layout, srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
                                       image = state.image, subresourceRange = VkImageSubresourceRange(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, levelCount = state.levels, layerCount = 1))
    let stages = VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT
    vkCmdPipelineBarrier(command, stages, stages, 0, 0, none, 0, none, 1, &barrier)
    state.layout = layout

## Every access an image transition orders, compute stores included.
let vulkan_image_access: u32 = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT

struct VulkanStaging:
    var buffer: u64
    var block: u64

## A buffer the CPU maps, of its own: an upload too large for the ring.
func vulkan_staging(device: VulkanDevice*, bytes: usize, usage: u32) -> VulkanStaging!:
    var staging: VulkanStaging
    errdefer vulkan_staging_destroy(device, staging)
    var info = VkBufferCreateInfo(sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, size = (u64)bytes, usage = usage)
    try vulkan_check(vkCreateBuffer(device.device, &info, none, &staging.buffer), "a Vulkan staging buffer could not be created")
    var requirements: VkMemoryRequirements
    vkGetBufferMemoryRequirements(device.device, staging.buffer, &requirements)
    staging.block = try vulkan_memory(device, requirements, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)
    try vulkan_check(vkBindBufferMemory(device.device, staging.buffer, staging.block, 0), "Vulkan staging memory could not be bound")
    return staging

func vulkan_staging_destroy(device: VulkanDevice*, staging: VulkanStaging):
    if staging.buffer != 0: vkDestroyBuffer(device.device, staging.buffer, none)
    if staging.block != 0: vkFreeMemory(device.device, staging.block, none)

func vulkan_copy_region(region: Region) -> VkBufferImageCopy:
    return VkBufferImageCopy(imageSubresource = VkImageSubresourceLayers(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, layerCount = 1),
                             imageOffset = VkOffset3D(x = (i32)region.x, y = (i32)region.y, z = 0),
                             imageExtent = VkExtent3D(width = region.width, height = region.height, depth = 1))

## Record the upload of `pixels` into `region` and submit it without waiting,
## staged through the upload ring (or, when larger than a ring slot, a buffer of
## its own freed when the submission completes).
func vulkan_texture_upload(device: VulkanDevice*, state: VulkanTexture*, pixels: const u8[], region: Region) -> !:
    if (u64)pixels.length <= vulkan_upload_slot_bytes:
        try vulkan_drawing(device)
        let ring = try vulkan_upload_room(device, (u64)pixels.length)
        let base = ring.mapped else trap("a mapped Vulkan ring slot lost its mapping")
        memory.copy(u8[]((u8*)base + (usize)ring.used, pixels.length), pixels, pixels.length)
        let offset = ring.used
        ring.used += (u64)pixels.length
        let slot = try vulkan_begin(device)
        vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
        var copy = vulkan_copy_region(region)
        copy.bufferOffset = offset
        vkCmdCopyBufferToImage(slot.command, ring.buffer, state.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy)
        vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
        let serial = try vulkan_end(device, slot)
        state.last_use = serial
        ring.serial = serial
        return
    var staging = try vulkan_staging(device, pixels.length, VK_BUFFER_USAGE_TRANSFER_SRC_BIT)
    defer vulkan_staging_destroy(device, staging)
    var mapped: void*? = none
    try vulkan_check(vkMapMemory(device.device, staging.block, 0, (u64)pixels.length, 0, &mapped), "Vulkan staging memory could not be mapped")
    let destination = mapped else error(failed, "Vulkan returned an empty memory mapping")
    memory.copy(u8[]((u8*)destination, pixels.length), pixels, pixels.length)
    vkUnmapMemory(device.device, staging.block)
    let slot = try vulkan_begin(device)
    vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
    var copy = vulkan_copy_region(region)
    vkCmdCopyBufferToImage(slot.command, staging.buffer, state.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy)
    vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    try vulkan_retire(slot, VulkanRetired(kind = vulkan_retired_buffer, handle = staging.buffer, allocation = VulkanAllocation(memory = staging.block)))
    staging = VulkanStaging()
    state.last_use = try vulkan_end(device, slot)

## Copy `region` of `source` to (`x`, `y`) of `target` and submit without
## waiting; both images are ready for sampling again after it.
func vulkan_texture_copy(device: VulkanDevice*, source: VulkanTexture*, region: Region, target: VulkanTexture*, x: u32, y: u32) -> !:
    let slot = try vulkan_begin(device)
    vulkan_transition(slot.command, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL)
    vulkan_transition(slot.command, target, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
    let layers = VkImageSubresourceLayers(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, layerCount = 1)
    var copy = VkImageCopy(srcSubresource = layers, srcOffset = VkOffset3D(x = (i32)region.x, y = (i32)region.y, z = 0),
                           dstSubresource = layers, dstOffset = VkOffset3D(x = (i32)x, y = (i32)y, z = 0),
                           extent = VkExtent3D(width = region.width, height = region.height, depth = 1))
    vkCmdCopyImage(slot.command, source.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy)
    vulkan_transition(slot.command, source, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    vulkan_transition(slot.command, target, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    let serial = try vulkan_end(device, slot)
    source.last_use = serial
    target.last_use = serial

## Fill levels 1.. of a mipmapped image from the one before each, by linear
## blits halving it, and submit without waiting; the image is ready for
## sampling again after.
func vulkan_texture_mipmaps(device: VulkanDevice*, state: VulkanTexture*) -> !:
    let slot = try vulkan_begin(device)
    vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
    var width = (i32)state.width
    var height = (i32)state.height
    for level in 1..<state.levels:
        # Level - 1 was written (uploaded, drawn or blitted): it becomes the source.
        vulkan_level_barrier(slot.command, state, level - 1, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL)
        let next_width = width // 2 if width > 1 else 1
        let next_height = height // 2 if height > 1 else 1
        var blit = VkImageBlit(srcSubresource = VkImageSubresourceLayers(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, mipLevel = level - 1, layerCount = 1),
                               dstSubresource = VkImageSubresourceLayers(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, mipLevel = level, layerCount = 1))
        blit.srcOffsets[1] = VkOffset3D(x = width, y = height, z = 1)
        blit.dstOffsets[1] = VkOffset3D(x = next_width, y = next_height, z = 1)
        vkCmdBlitImage(slot.command, state.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, state.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_LINEAR)
        width = next_width
        height = next_height
    # Every level but the last is a source now; the last was written last.
    for level in 0..<state.levels - 1:
        vulkan_level_barrier(slot.command, state, level, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    vulkan_level_barrier(slot.command, state, state.levels - 1, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
    state.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
    state.last_use = try vulkan_end(device, slot)

## A barrier moving one mip level of the image between layouts.
func vulkan_level_barrier(command: u64, state: VulkanTexture*, level: u32, from_layout: u32, to_layout: u32):
    var barrier = VkImageMemoryBarrier(sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
                                       srcAccessMask = vulkan_image_access, dstAccessMask = vulkan_image_access,
                                       oldLayout = from_layout, newLayout = to_layout, srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
                                       image = state.image, subresourceRange = VkImageSubresourceRange(aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, baseMipLevel = level, levelCount = 1, layerCount = 1))
    let stages = VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT
    vkCmdPipelineBarrier(command, stages, stages, 0, 0, none, 0, none, 1, &barrier)

## The device's sampler for `filter` on an image the device filters linearly
## or not.
func vulkan_sampler_for(device: VulkanDevice*, filter: Filter, filterable: bool) -> u64:
    if not filterable or filter == Filter.nearest:
        return device.nearest
    return device.trilinear if filter == Filter.trilinear else device.linear

## Destroy a framebuffer this call still owns; zero once it has been retired.
func vulkan_framebuffer_release(device: VulkanDevice*, framebuffer: u64):
    if framebuffer != 0: vkDestroyFramebuffer(device.device, framebuffer, none)

## Render `canvas` into the image cleared to `color` (over its contents when
## `keep`) and submit it without waiting. The pass leaves the image ready for
## sampling; its framebuffer and uploads are freed when the submission completes.
func vulkan_texture_render(device: VulkanDevice*, state: VulkanTexture*, canvas: Canvas, color: Color, keep: bool) -> !:
    let formats = try vulkan_format_state(device, state.native_format)
    let depth = try vulkan_depth_for(device, state.width, state.height)
    var attachments: u64[2] = [state.attachment if state.levels > 1 else state.view, depth]
    var framebuffer_info = VkFramebufferCreateInfo(sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
                                                   renderPass = formats.pass_texture, attachmentCount = 2, pAttachments = &attachments[0], width = state.width, height = state.height, layers = 1)
    var framebuffer: u64 = 0
    try vulkan_check(vkCreateFramebuffer(device.device, &framebuffer_info, none, &framebuffer), "a Vulkan framebuffer could not be created")
    defer vulkan_framebuffer_release(device, framebuffer)
    if reads_scene_depth_anywhere(&canvas): try vulkan_depth_copy_for(device, state.width, state.height)
    var upload = try vulkan_upload(device, canvas)
    defer vulkan_upload_destroy(device, upload)
    let slot = try vulkan_begin(device)
    let serial = device.submitted + 1
    var pass = formats.pass_texture
    if state.levels > 1:
        # The pass moves only the first level (the framebuffer's view): the
        # others are made ready for sampling here, and the first as the pass
        # starts it.
        vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
        if keep:
            vulkan_level_barrier(slot.command, state, 0, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL)
            pass = formats.pass_load
    elif keep:
        # The loading pass starts from the attachment layout, made ready outside it.
        vulkan_transition(slot.command, state, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL)
        pass = formats.pass_load
    else:
        # The pass begins from an undefined layout and clears; only the bookkeeping moves.
        state.layout = VK_IMAGE_LAYOUT_UNDEFINED
    try vulkan_encode(device, slot.command, formats, pass, framebuffer, canvas, upload, color, state.width, state.height, serial, depth_image = device.depth_image)
    state.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
    try vulkan_retire(slot, VulkanRetired(kind = vulkan_retired_framebuffer, handle = framebuffer))
    framebuffer = 0
    try vulkan_upload_retire(slot, &upload)
    state.last_use = try vulkan_end(device, slot)