dymokomi/luce-base
Source of release 0.43.0
0.43.0 / src / back / native / x86_64 / transfers.lucb
#==============================================================================================
#
# x86_64/transfers - Eightbyte transfers
#
# DESCRIPTION:
# Aggregates moved between memory and registers an eightbyte at a time, as the
# System V and Windows conventions classify them.
#
#==============================================================================================
extend Generator:
# mark: Eightbyte transfers ================================================================
## Exactly `width` bytes (1..8) at `offset(base)` into the 64-bit register `reg`, zero
## above; `%rax` helps with the odd widths, or `%r11` when `reg` is `%rax` itself, so an
## argument's load leaves the callee parked in r11 alone. The three bytes above the low
## four of a seven-byte width come from an overlapping four-byte load shifted down, not
## in two pieces of their own, which would need a second helper (r11 again).
func load_bytes(reg: str, base: str, offset: u64, width: u64) -> !:
let l = narrow_name(reg)
if width == 8:
try self.offset_move("movq ", offset, "(")
try self.writer.put(base)
try self.writer.put("), ")
try self.writer.put(reg)
return
if width == 4 or width == 2 or width == 1:
try self.offset_move("movl " if width == 4 else ("movzwl " if width == 2 else "movzbl "), offset, "(")
try self.writer.put(base)
try self.writer.put("), ")
try self.writer.put(l)
return
# 3, 5, 6, 7: the low part, then the rest shifted into place
let low: u64 = 4 if width > 4 else 2
let helper = "%r11" if reg == "%rax" else "%rax"
if width == 7:
try self.offset_move("movl ", offset + 3, "(")
try self.writer.put(base)
try self.writer.put("), ")
try self.writer.put(narrow_name(helper))
try self.writer.line3("shrl $8, ", narrow_name(helper), "")
else:
try self.load_bytes(helper, base, offset + low, width - low)
try self.load_bytes(reg, base, offset, low)
try self.writer.line("shlq $")
try self.writer.put_u64(low * 8)
try self.writer.put(", ")
try self.writer.put(helper)
try self.writer.line3("orq ", helper, ", ")
try self.writer.put(reg)
## Exactly `width` bytes of the 64-bit register `reg` to `offset(base)`; the register is
## consumed.
func store_bytes(reg: str, base: str, offset: u64, width: u64) -> !:
if width == 8 or width == 4 or width == 2 or width == 1:
try self.writer.line(store_word_mnemonic(width))
try self.writer.put(register_of_width(reg, width))
try self.writer.put(", ")
try self.writer.put_u64(offset)
try self.writer.put("(")
try self.writer.put(base)
try self.writer.put(")")
return
let low: u64 = 4 if width > 4 else 2
try self.store_bytes(reg, base, offset, low)
try self.writer.line("shrq $")
try self.writer.put_u64(low * 8)
try self.writer.put(", ")
try self.writer.put(reg)
try self.store_bytes(reg, base, offset + low, width - low)
## Eightbyte `k` of an aggregate of `size` bytes at (`base`), into `reg`: an integer
## register, or an SSE one through rax.
func load_eightbyte(reg: str, base: str, k: u32, size: u32, sse: bool) -> !:
let offset = (u64)k * 8
let width = (u64)size - offset if (u64)size - offset < 8 else 8
if sse:
if width == 8 or width == 4:
# A preceding INTEGER result may already be in rax. Loading an SSE
# eightbyte must not overwrite that other half of a mixed return.
try self.based_move("movq " if width == 8 else "movd ", offset, base, ", ")
try self.writer.put(reg)
else:
# rax kept in the third float scratch register, which no argument takes
let keep = self.reg(ir.Class.d, 2)
try self.writer.line3("movq %rax, ", keep, "")
try self.load_bytes("%rax", base, offset, width)
try self.writer.line3("movq %rax, ", reg, "")
try self.writer.line3("movq ", keep, ", %rax")
else:
try self.load_bytes(reg, base, offset, width)
## Eightbyte `k` of an aggregate of `size` bytes, from `reg` to (%r10).
func store_eightbyte(reg: str, k: u32, size: u32, sse: bool) -> !:
let offset = (u64)k * 8
let width = (u64)size - offset if (u64)size - offset < 8 else 8
if sse:
# rax may hold the INTEGER half of a mixed SSE/INTEGER return.
# Store common widths directly; use non-ABI scratch for short tails.
if width == 8 or width == 4:
try self.writer.line3("movq " if width == 8 else "movd ", reg, ", ")
try self.writer.put_u64(offset)
try self.writer.put("(%r10)")
else:
try self.writer.line3("movq ", reg, ", %r11")
try self.store_bytes("%r11", "%r10", offset, width)
else:
try self.store_bytes(reg, "%r10", offset, width)
## Store the incoming arguments into their slots.
## By slot, the parameters `receive_params` moves straight from their argument registers
## into their entry loads' results (`incoming`, System V only): their slots are never
## written, so they need no home. It walks the registers as `receive_params` does, for a
## function whose parameters are all scalars; otherwise every slot keeps its home.
func straight_params(entry: u32[]) -> bool[]!:
var straight = try new bool[self.f.slots.count + 1] ---
for k in 0..<straight.length:
straight[k] = false
for p in self.f.params.span():
if p.size != 0:
# an aggregate is copied into its slot first, by a call past 128 bytes, which
# leaves no argument register as it was
return straight
if self.convention.windows:
# by position: the first four arguments, integer or float, in their registers
let hidden: usize = 1 if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private) else 0
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
if i + hidden < 4 and p.size == 0 and p.cls != ir.Class.h and entry[i] != 0 and self.fr.slot_regs.get((usize)p.slot) == 0:
straight[(usize)p.slot] = true
return straight
var next_int: u32 = 1 if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private) else 0
var next_float: u32 = 0
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
let moved = self.fr.slot_regs.get((usize)p.slot) == 0 and entry[i] != 0 and p.cls != ir.Class.h
if ir.is_float_class(p.cls):
if next_float < 8:
straight[(usize)p.slot] = moved
next_float += 1
elif next_int < 6:
straight[(usize)p.slot] = moved
next_int += 1
return straight
func receive_params(entry: u32[], moves: list.List[incoming.Move]*) -> !:
if self.convention.windows:
try self.receive_windows(entry, moves)
return
var next_int: u32 = 1 if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private) else 0
var next_float: u32 = 0
var stack_offset: u64 = 16
# a memory argument copied by memcpy (past 128 bytes) would clobber the ones left
# in their registers for `receive_directly`
var direct = true
for p in self.f.params.span():
if p.size > 128:
direct = false
var i: usize = 0
while i < self.f.params.count:
let p = self.f.params.get(i)
let home = self.slot_home(p.slot)
if p.size == 0:
let code = (usize)self.fr.slot_regs.get((usize)p.slot)
let straight = direct and code == 0 and entry[i] != 0 and p.cls != ir.Class.h
if ir.is_float_class(p.cls):
if next_float < 8 and straight:
try moves.push(incoming.Move(at = entry[i] - 1, argument = u8(next_float), float = true))
next_float += 1
elif next_float < 8:
if p.cls == ir.Class.h:
try self.writer.line2("movd ", float_argument_names[(usize)next_float])
try self.writer.put(", %eax")
try self.writer.line("movw %ax, ")
try self.frame_operand(home)
else:
try self.frame_store(float_argument_names[(usize)next_float], p.cls, home)
next_float += 1
else:
try self.stack_param_load("%rax", stack_offset)
try self.writer.line(store_word_mnemonic(2 if p.cls == ir.Class.h else (4 if p.cls == ir.Class.s else 8)))
try self.writer.put(register_of_width("%rax", 2 if p.cls == ir.Class.h else (4 if p.cls == ir.Class.s else 8)))
try self.writer.put(", ")
try self.frame_operand(home)
stack_offset += 8
elif next_int < 6 and straight:
try moves.push(incoming.Move(at = entry[i] - 1, argument = u8(next_int), float = false))
next_int += 1
elif next_int < 6:
let source_name = argument_names[(usize)next_int] if p.cls == ir.Class.l else narrow_name(argument_names[(usize)next_int])
if code != 0:
try self.writer.line4("movq " if p.cls == ir.Class.l else "movl ", source_name, ", ", self.class_name((u8)code, p.cls))
else:
try self.frame_store(source_name, p.cls, home)
next_int += 1
else:
try self.stack_param_load("%rax", stack_offset)
if code != 0:
try self.writer.line4("movq " if p.cls == ir.Class.l else "movl ", "%rax" if p.cls == ir.Class.l else "%eax", ", ", self.class_name((u8)code, p.cls))
else:
try self.frame_store("%rax" if p.cls == ir.Class.l else "%eax", p.cls, home)
stack_offset += 8
else:
let sse = self.convention.sse_of(p.cls, p.size, p.shape)
let words: u32 = 2 if p.size > 8 else 1
let sse_count = count_bits(sse)
let int_count = words - sse_count
if not self.convention.in_memory(p.size, p.shape) and next_int + int_count <= 6 and next_float + sse_count <= 8:
try self.frame_address("%r10", home)
var k: u32 = 0
while k < words:
if (sse & (u8)(1 << k)) != 0:
try self.store_eightbyte(float_argument_names[(usize)next_float], k, p.size, true)
next_float += 1
else:
try self.store_eightbyte(argument_names[(usize)next_int], k, p.size, false)
next_int += 1
k += 1
else:
# copied from the caller's stack area in a second pass, after every
# register is safely stored
stack_offset += round_up((u64)p.size, 8)
i += 1
try self.copy_memory_params()
## The aggregates passed in the caller's stack area copied into their slots, after
## every register argument is safely stored: the same walk over the arguments.
func copy_memory_params() -> !:
var stack_offset: u64 = 16
var next_int: u32 = 1 if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private) else 0
var next_float: u32 = 0
var i: usize = 0
while i < self.f.params.count:
let p = self.f.params.get(i)
if p.size == 0:
if ir.is_float_class(p.cls):
if next_float < 8:
next_float += 1
else:
stack_offset += 8
elif next_int < 6:
next_int += 1
else:
stack_offset += 8
else:
let sse = self.convention.sse_of(p.cls, p.size, p.shape)
let words: u32 = 2 if p.size > 8 else 1
let sse_count = count_bits(sse)
let int_count = words - sse_count
if not self.convention.in_memory(p.size, p.shape) and next_int + int_count <= 6 and next_float + sse_count <= 8:
next_int += int_count
next_float += sse_count
else:
try self.writer.line("leaq ")
try self.writer.put_u64(stack_offset)
try self.writer.put("(%rbp), %r10")
try self.frame_address("%r11", self.slot_home(p.slot))
try self.copy_to_r11("%r10", (u64)p.size)
stack_offset += round_up((u64)p.size, 8)
i += 1
func stack_param_load(reg: str, offset: u64) -> !:
try self.writer.line("movq ")
try self.writer.put_u64(offset + (self.fr.size if self.convention.windows else 0))
try self.writer.put("(%rbp), ")
try self.writer.put(reg)
## `size` bytes from (`origin`) to (%r11): inline up to 128, else `memcpy`.
func copy_to_r11(origin: str, size: u64) -> !:
if size <= 128:
try self.inline_copy(origin, "%r11", size)
return
try self.writer.line3("movq ", origin, ", ")
try self.writer.put(self.c_argument(1))
try self.writer.line3("movq %r11, ", self.c_argument(0), "")
try self.load_immediate(self.c_argument(2), (i64)size)
try self.call_symbol("memcpy")
## The register of a C routine's integer argument `k` (0, 1, 2) under the convention:
## rdi, rsi, rdx on System V; rcx, rdx, r8 on Windows.
func c_argument(k: u32) -> str:
if self.convention.windows:
return windows_arguments[(usize)k]
return argument_names[(usize)k]
## A call of a C routine whose arguments are in place (`c_argument`): on Windows with the
## shadow area its callee may write.
func call_symbol(name: str) -> !:
if self.convention.windows:
try self.writer.line("subq $32, %rsp")
try self.writer.line("call ")
try self.symbol(name)
if self.convention.windows:
try self.writer.line("addq $32, %rsp")
else:
try self.writer.put("@PLT")