dymokomi/luce-gpu
Source of release 0.5.7
0.5.7 / src / gpu / metal / copies.lucb
#==============================================================================================
#
# copies - Metal uploads, copies, readback and mipmaps with render passes
#
# DESCRIPTION:
# Drawing work never uses a blit or compute encoder on Metal: the driver keeps about 160 MB
# for the rest of a process once it first runs either, on top of the 230 MB a render pass
# costs. Texture uploads, texture copies, mipmap levels, buffer uploads and fills, and
# readback are render passes instead. A pass into the texture's region (through an unorm
# view of an sRGB texture, so bytes are written as they are) reads texels from the upload
# ring, another texture or the level above; readback and buffer work are passes with no
# attachments whose fragments write a buffer. Explicit compute (Compute passes, ray
# builds) still uses compute and acceleration structure encoders.
#
#==============================================================================================
## The shaders: a full-viewport triangle and one fragment function per job.
let metal_copy_source: c.str = "#include <metal_stdlib>\nusing namespace metal;\n\n// origin: the target region's corner; source: the source region's corner;\n// row: bytes per buffer row; offset: the buffer's first byte; kind: the texel\n// layout; count and value: a word copy's or fill's length and value; size: a\n// mip level's size; width: texels a buffer pass covers per row.\nstruct CopyParams { uint2 origin; uint2 source; uint row; uint offset; uint kind; uint count; uint value; uint width; float2 size; };\n\nvertex float4 luce_copy_vertex(uint id [[vertex_id]]) {\n float2 p = float2(float((id << 1) & 2), float(id & 2));\n return float4(p * 2.0 - 1.0, 0.0, 1.0);\n}\n\n// Kinds: 0 four unorm bytes, 1 one unorm byte, 2 four halves, 3 four floats,\n// 4 one float, 6 two floats; 5 one uint (the _uint functions).\nfragment float4 luce_upload(float4 at [[position]], constant CopyParams& p [[buffer(0)]], const device uchar* bytes [[buffer(1)]]) {\n uint2 xy = uint2(at.xy) - p.origin;\n const device uchar* row = bytes + p.offset + xy.y * p.row;\n switch (p.kind) {\n case 0: return float4(((const device uchar4*)row)[xy.x]) / 255.0;\n case 1: return float4(float(row[xy.x]) / 255.0, 0.0, 0.0, 1.0);\n case 2: return float4(((const device half4*)row)[xy.x]);\n case 3: return ((const device float4*)row)[xy.x];\n case 4: return float4(((const device float*)row)[xy.x], 0.0, 0.0, 1.0);\n default: return float4(((const device float2*)row)[xy.x], 0.0, 1.0);\n }\n}\n\nfragment uint4 luce_upload_uint(float4 at [[position]], constant CopyParams& p [[buffer(0)]], const device uchar* bytes [[buffer(1)]]) {\n uint2 xy = uint2(at.xy) - p.origin;\n return uint4(((const device uint*)(bytes + p.offset + xy.y * p.row))[xy.x], 0u, 0u, 1u);\n}\n\nfragment float4 luce_copy(float4 at [[position]], constant CopyParams& p [[buffer(0)]], texture2d<float, access::read> image [[texture(0)]]) {\n return image.read(uint2(at.xy) - p.origin + p.source);\n}\n\nfragment uint4 luce_copy_uint(float4 at [[position]], constant CopyParams& p [[buffer(0)]], texture2d<uint, access::read> image [[texture(0)]]) {\n return image.read(uint2(at.xy) - p.origin + p.source);\n}\n\n// One level from the level above: a bilinear sample at the texel's centre\n// averages the four texels it covers.\nfragment float4 luce_mip(float4 at [[position]], constant CopyParams& p [[buffer(0)]], texture2d<float> above [[texture(0)]], sampler smooth [[sampler(0)]]) {\n return above.sample(smooth, at.xy / p.size, level(0.0));\n}\n\n// Readback: no attachments; each fragment writes its texel's bytes.\nfragment void luce_read(float4 at [[position]], constant CopyParams& p [[buffer(0)]], device uchar* bytes [[buffer(1)]], texture2d<float, access::read> image [[texture(0)]]) {\n uint2 xy = uint2(at.xy);\n float4 v = image.read(xy + p.source);\n device uchar* row = bytes + p.offset + xy.y * p.row;\n switch (p.kind) {\n case 0: ((device uchar4*)row)[xy.x] = uchar4(rint(clamp(v, 0.0, 1.0) * 255.0)); break;\n case 1: row[xy.x] = uchar(rint(clamp(v.x, 0.0, 1.0) * 255.0)); break;\n case 2: ((device half4*)row)[xy.x] = half4(v); break;\n case 3: ((device float4*)row)[xy.x] = v; break;\n case 4: ((device float*)row)[xy.x] = v.x; break;\n default: ((device float2*)row)[xy.x] = v.xy; break;\n }\n}\n\nfragment void luce_read_uint(float4 at [[position]], constant CopyParams& p [[buffer(0)]], device uchar* bytes [[buffer(1)]], texture2d<uint, access::read> image [[texture(0)]]) {\n uint2 xy = uint2(at.xy);\n ((device uint*)(bytes + p.offset + xy.y * p.row))[xy.x] = image.read(xy + p.source).x;\n}\n\n// The depth attachment as r32float, for draws that read the scene depth.\nfragment float4 luce_depth(float4 at [[position]], depth2d<float, access::read> depth [[texture(0)]]) {\n return float4(depth.read(uint2(at.xy)), 0.0, 0.0, 1.0);\n}\n\n// Words: each fragment copies one 32-bit word, or (kind 1) sets it to value.\nfragment void luce_words(float4 at [[position]], constant CopyParams& p [[buffer(0)]], const device uint* source [[buffer(1)]], device uint* target [[buffer(2)]]) {\n uint index = uint(at.y) * p.width + uint(at.x);\n if (index < p.count)\n target[index] = p.kind == 1u ? p.value : source[index];\n}\n"
## The parameters every copy shader reads (`CopyParams`), 48 bytes.
struct MetalCopyParams:
var origin_x: u32
var origin_y: u32
var source_x: u32
var source_y: u32
var row: u32
var offset: u32
var kind: u32
var count: u32
var value: u32
var width: u32
var size_x: f32
var size_y: f32
## Pipelines by fragment function and attachment format, built on first use.
struct MetalCopyPipeline:
var function: u8
var format: u64
var pipeline: void*?
struct MetalCopies:
var library: void*?
var pipelines: MetalCopyPipeline[32]
var count: usize
var linear: void*?
let metal_copy_upload: u8 = 0
let metal_copy_upload_uint: u8 = 1
let metal_copy_copy: u8 = 2
let metal_copy_copy_uint: u8 = 3
let metal_copy_mip: u8 = 4
let metal_copy_read: u8 = 5
let metal_copy_read_uint: u8 = 6
let metal_copy_word_fill: u8 = 7
let metal_copy_depth: u8 = 8
## No attachment: the readback and buffer passes.
let metal_copy_no_format: u64 = 0
func metal_copy_function(function: u8) -> c.str:
if function == metal_copy_upload: return "luce_upload"
if function == metal_copy_upload_uint: return "luce_upload_uint"
if function == metal_copy_copy: return "luce_copy"
if function == metal_copy_copy_uint: return "luce_copy_uint"
if function == metal_copy_mip: return "luce_mip"
if function == metal_copy_read: return "luce_read"
if function == metal_copy_read_uint: return "luce_read_uint"
if function == metal_copy_depth: return "luce_depth"
return "luce_words"
## The texel layout the shaders read and write for `format` (CopyParams.kind).
func metal_copy_kind(pixel_format: Format) -> u32:
if pixel_format == Format.r8: return 1
if pixel_format == Format.rgba16_float: return 2
if pixel_format == Format.rgba32_float: return 3
if pixel_format == Format.r32_float: return 4
if pixel_format == Format.r32_uint: return 5
if pixel_format == Format.rg32_float: return 6
return 0
func metal_copies(device: MetalDevice*) -> MetalCopies*!:
if let copies = device.copies: return copies
let copies = try new MetalCopies() in memory.heap
errdefer metal_copies_destroy(copies)
# MSL 3.0: fragments that write device buffers.
copies.library = try metal_compile(metal_required(device.device), metal_copy_source, false, 3u64 << 16)
copies.linear = try metal_sampler(metal_required(device.device), 1)
device.copies = copies
return copies
func metal_copies_destroy(copies: MetalCopies*):
for entry in copies.pipelines[..<copies.count]:
metal_release_optional(entry.pipeline)
metal_release_optional(copies.library)
metal_release_optional(copies.linear)
free(copies) in memory.heap
## The pipeline of `function` drawing into attachments of `format` (none: no attachment).
func metal_copy_pipeline(device: MetalDevice*, copies: MetalCopies*, function: u8, format: u64) -> void*!:
for entry in copies.pipelines[..<copies.count]:
if entry.function == function and entry.format == format:
return metal_required(entry.pipeline)
if copies.count == copies.pipelines.length:
error(failed, "too many Metal copy pipelines")
let library = metal_required(copies.library)
let vertex = metal_with_object(library, metal_sel("newFunctionWithName:"), try metal_text("luce_copy_vertex")) else error(failed, "the copy vertex stage is missing")
defer metal_void(vertex, metal_sel("release"))
let fragment = metal_with_object(library, metal_sel("newFunctionWithName:"), try metal_text(metal_copy_function(function))) else error(failed, "a copy fragment stage is missing")
defer metal_void(fragment, metal_sel("release"))
let descriptor = metal_ptr(try metal_class("MTLRenderPipelineDescriptor"), metal_sel("new")) else error(failed, "could not create a GPU pipeline descriptor")
defer metal_void(descriptor, metal_sel("release"))
metal_object(descriptor, metal_sel("setVertexFunction:"), vertex)
metal_object(descriptor, metal_sel("setFragmentFunction:"), fragment)
if format != metal_copy_no_format:
let attachments = metal_required(metal_ptr(descriptor, metal_sel("colorAttachments")))
let attachment = metal_required(metal_index(attachments, metal_sel("objectAtIndexedSubscript:"), 0))
metal_uint_arg(attachment, metal_sel("setPixelFormat:"), format)
var failure: void*? = none
let pipeline = metal_pipeline(metal_required(device.device), metal_sel("newRenderPipelineStateWithDescriptor:error:"), descriptor, &failure) else error(failed, "could not build a Metal copy pipeline")
copies.pipelines[copies.count] = MetalCopyPipeline(function = function, format = format, pipeline = pipeline)
copies.count += 1
return pipeline
## A view of `texture` that reads and writes bytes as stored: an sRGB texture
## through its unorm format, others as they are (+1, the caller releases it).
func metal_raw_view(texture: MetalTexture*) -> void*!:
let native = metal_required(texture.texture)
let format = 70u64 if texture.format == Format.rgba8 else metal_pixel_format(texture.format)
return metal_with_u64(native, metal_sel("newTextureViewWithPixelFormat:"), format) else error(failed, "could not view a Metal texture")
## A render pass of `command` into `level` of `target` (a view or texture of
## `format`), loading what is there, drawing only `region`.
func metal_copy_pass(command: void*, target: void*, level: u64, region: Region) -> void*!:
let descriptor = metal_ptr(try metal_class("MTLRenderPassDescriptor"), metal_sel("renderPassDescriptor")) else error(failed, "could not create a Metal render pass")
let attachments = metal_required(metal_ptr(descriptor, metal_sel("colorAttachments")))
let attachment = metal_required(metal_index(attachments, metal_sel("objectAtIndexedSubscript:"), 0))
metal_object(attachment, metal_sel("setTexture:"), target)
metal_uint_arg(attachment, metal_sel("setLevel:"), level)
metal_uint_arg(attachment, metal_sel("setLoadAction:"), 1)
metal_uint_arg(attachment, metal_sel("setStoreAction:"), 1)
let encoder = metal_with_object(command, metal_sel("renderCommandEncoderWithDescriptor:"), descriptor) else error(failed, "could not create a Metal render encoder")
metal_copy_region(encoder, region)
return encoder
## A render pass of `command` with no attachments, `width` by `height` fragments.
func metal_buffer_pass(command: void*, width: u32, height: u32) -> void*!:
let descriptor = metal_ptr(try metal_class("MTLRenderPassDescriptor"), metal_sel("renderPassDescriptor")) else error(failed, "could not create a Metal render pass")
metal_uint_arg(descriptor, metal_sel("setRenderTargetWidth:"), (u64)width)
metal_uint_arg(descriptor, metal_sel("setRenderTargetHeight:"), (u64)height)
metal_uint_arg(descriptor, metal_sel("setDefaultRasterSampleCount:"), 1)
let encoder = metal_with_object(command, metal_sel("renderCommandEncoderWithDescriptor:"), descriptor) else error(failed, "could not create a Metal render encoder")
metal_copy_region(encoder, Region(0, 0, width, height))
return encoder
func metal_copy_region(encoder: void*, region: Region):
metal_viewport(encoder, metal_sel("setViewport:"), LuceGpuMetalViewport(x = (f64)region.x, y = (f64)region.y, width = (f64)region.width, height = (f64)region.height, near = 0.0, far = 1.0))
metal_scissor(encoder, metal_sel("setScissorRect:"), LuceGpuMetalScissor(x = (u64)region.x, y = (u64)region.y, width = (u64)region.width, height = (u64)region.height))
## Draw the full-viewport triangle with `pipeline` and `params`, and end the pass.
func metal_copy_draw(encoder: void*, pipeline: void*, params: MetalCopyParams):
var bytes = params
metal_object(encoder, metal_sel("setRenderPipelineState:"), pipeline)
metal_inline_bytes(encoder, metal_sel("setFragmentBytes:length:atIndex:"), &bytes, 48, 0)
metal_draw(encoder, metal_sel("drawPrimitives:vertexStart:vertexCount:"), 3, 0, 3)
metal_void(encoder, metal_sel("endEncoding"))
# mark: Textures ===============================================================================
## Write `region` of `texture` from tightly packed texels at `span` in the upload ring.
func metal_copy_upload_texture(device: MetalDevice*, command: void*, texture: MetalTexture*, region: Region, buffer: void*, offset: u64) -> !:
let copies = try metal_copies(device)
let view = try metal_raw_view(texture)
defer metal_void(view, metal_sel("release"))
let integer = texture.format == Format.r32_uint
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_upload_uint if integer else metal_copy_upload, metal_uint(view, metal_sel("pixelFormat")))
let encoder = try metal_copy_pass(command, view, 0, region)
metal_bind_buffer(encoder, metal_sel("setFragmentBuffer:offset:atIndex:"), buffer, offset, 1)
metal_copy_draw(encoder, pipeline, MetalCopyParams(origin_x = region.x, origin_y = region.y, row = region.width * (u32)texel_bytes(texture.format), kind = metal_copy_kind(texture.format)))
## Copy `region` of `source` to (`x`, `y`) of `target` (the same format).
func metal_copy_texture_pass(device: MetalDevice*, command: void*, source: MetalTexture*, region: Region, target: MetalTexture*, x: u32, y: u32) -> !:
let copies = try metal_copies(device)
let from_view = try metal_raw_view(source)
defer metal_void(from_view, metal_sel("release"))
let to_view = try metal_raw_view(target)
defer metal_void(to_view, metal_sel("release"))
let integer = target.format == Format.r32_uint
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_copy_uint if integer else metal_copy_copy, metal_uint(to_view, metal_sel("pixelFormat")))
let encoder = try metal_copy_pass(command, to_view, 0, Region(x, y, region.width, region.height))
metal_bind_texture(encoder, metal_sel("setFragmentTexture:atIndex:"), from_view, 0)
metal_copy_draw(encoder, pipeline, MetalCopyParams(origin_x = x, origin_y = y, source_x = region.x, source_y = region.y))
## Fill every level after the first from the one above, a render pass each.
func metal_copy_mipmaps(device: MetalDevice*, command: void*, texture: MetalTexture*, levels: u32) -> !:
let copies = try metal_copies(device)
let native = metal_required(texture.texture)
let format = metal_uint(native, metal_sel("pixelFormat"))
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_mip, format)
var width = texture.width
var height = texture.height
for level in 1..<levels:
width = width // 2 if width > 1 else 1
height = height // 2 if height > 1 else 1
# The level above alone, so the pass never samples what it writes.
let above = metal_level_view(native, format, (u64)level - 1) else error(failed, "could not view a Metal texture level")
let encoder = try metal_copy_pass(command, native, (u64)level, Region(0, 0, width, height))
metal_bind_texture(encoder, metal_sel("setFragmentTexture:atIndex:"), above, 0)
metal_bind_texture(encoder, metal_sel("setFragmentSamplerState:atIndex:"), metal_required(copies.linear), 0)
metal_copy_draw(encoder, pipeline, MetalCopyParams(size_x = (f32)width, size_y = (f32)height))
metal_void(above, metal_sel("release"))
## Copy the depth attachment `depth` into `copy` (r32float, as large), both
## `width` by `height`: what draws reading the scene depth sample.
func metal_copy_depth_texture(device: MetalDevice*, command: void*, copies: MetalCopies*, depth: void*, copy: void*, width: u64, height: u64) -> !:
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_depth, 55)
let encoder = try metal_copy_pass(command, copy, 0, Region(0, 0, (u32)width, (u32)height))
metal_bind_texture(encoder, metal_sel("setFragmentTexture:atIndex:"), depth, 0)
metal_copy_draw(encoder, pipeline, MetalCopyParams())
## Read `region` of `texture` into `buffer` from `offset`, rows `row` bytes apart.
func metal_copy_read_texture(device: MetalDevice*, command: void*, texture: MetalTexture*, region: Region, buffer: void*, offset: u64, row: u64) -> !:
let copies = try metal_copies(device)
let view = try metal_raw_view(texture)
defer metal_void(view, metal_sel("release"))
let integer = texture.format == Format.r32_uint
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_read_uint if integer else metal_copy_read, metal_copy_no_format)
let encoder = try metal_buffer_pass(command, region.width, region.height)
metal_bind_texture(encoder, metal_sel("setFragmentTexture:atIndex:"), view, 0)
metal_bind_buffer(encoder, metal_sel("setFragmentBuffer:offset:atIndex:"), buffer, offset, 1)
metal_copy_draw(encoder, pipeline, MetalCopyParams(source_x = region.x, source_y = region.y, row = (u32)row, kind = metal_copy_kind(texture.format)))
# mark: Buffers ================================================================================
## Words a buffer pass covers per row, and the most rows it has.
let metal_words_width: u64 = 4096
let metal_words_rows: u64 = 16384
## Copy `bytes` (a multiple of four) from `source` at `source_offset` to
## `target` at `target_offset`, or, with no source, set every word to `value`.
func metal_copy_words(device: MetalDevice*, command: void*, source: void*?, source_offset: u64, target: void*, target_offset: u64, bytes: u64, value: u32) -> !:
let copies = try metal_copies(device)
let pipeline = try metal_copy_pipeline(device, copies, metal_copy_word_fill, metal_copy_no_format)
let most = metal_words_width * metal_words_rows
var done: u64 = 0
let words = bytes // 4
while done < words:
let count = most if words - done > most else words - done
let rows = (count + metal_words_width - 1) // metal_words_width
let encoder = try metal_buffer_pass(command, (u32)(count if count < metal_words_width else metal_words_width), (u32)rows)
metal_bind_buffer(encoder, metal_sel("setFragmentBuffer:offset:atIndex:"), source else target, source_offset + done * 4 if source != none else target_offset, 1)
metal_bind_buffer(encoder, metal_sel("setFragmentBuffer:offset:atIndex:"), target, target_offset + done * 4, 2)
metal_copy_draw(encoder, pipeline, MetalCopyParams(kind = 0 if source != none else 1, count = (u32)count, value = value, width = (u32)metal_words_width))
done += count
extern func metal_with_u64 as "objc_msgSend"(object: void*, selector: void*, value: u64) -> void*?
extern func metal_level_view_call as "objc_msgSend"(object: void*, selector: void*, format: u64, kind: u64, levels: LuceGpuMetalRange, slices: LuceGpuMetalRange) -> void*?
func metal_level_view(texture: void*, format: u64, level: u64) -> void*?:
# MTLTextureType2D (2).
return metal_level_view_call(texture, metal_sel("newTextureViewWithPixelFormat:textureType:levels:slices:"), format, 2, LuceGpuMetalRange(level, 1), LuceGpuMetalRange(0, 1))