dymokomi/luce-base
Source of release 0.41.0
0.41.0 / src / back / native / arm64 / functions.lucb
#==============================================================================================
#
# arm64/functions - Functions
#
# DESCRIPTION:
# Each function's prologue, parameters received under AAPCS64 and Apple's variant,
# body, and epilogue.
#
#==============================================================================================
extend Generator:
# mark: Functions ==========================================================================
## Copy `size` bytes from [`origin`] to [`to`] with plain loads and stores through x16 and
## x17: 16 at a time, then 8, 4, 2, 1. The addresses are records in frames or fields, so
## unaligned access is fine on arm64; a copy of up to 128 bytes is cheaper inline than as
## a call.
func inline_copy(origin: str, to: str, size: u64) -> !:
var done: u64 = 0
while size - done >= 16:
try self.pair_move("ldp x16, x17, [", origin, done)
try self.pair_move("stp x16, x17, [", to, done)
done += 16
let widths: u64[4] = [8, 4, 2, 1]
let loads: str[4] = ["ldr x16, [", "ldr w16, [", "ldrh w16, [", "ldrb w16, ["]
let stores: str[4] = ["str x16, [", "str w16, [", "strh w16, [", "strb w16, ["]
var k: usize = 0
while k < 4:
if size - done >= widths[k]:
try self.pair_move(loads[k], origin, done)
try self.pair_move(stores[k], to, done)
done += widths[k]
k += 1
## Zero `size` bytes at [`to`] the same way.
func inline_zero(to: str, size: u64) -> !:
var done: u64 = 0
while size - done >= 16:
try self.pair_move("stp xzr, xzr, [", to, done)
done += 16
let widths: u64[4] = [8, 4, 2, 1]
let stores: str[4] = ["str xzr, [", "str wzr, [", "strh wzr, [", "strb wzr, ["]
var k: usize = 0
while k < 4:
if size - done >= widths[k]:
try self.pair_move(stores[k], to, done)
done += widths[k]
k += 1
## `mnemonic` then `base, #offset]`.
func pair_move(mnemonic: str, base: str, offset: u64) -> !:
try self.writer.line(mnemonic)
try self.writer.put(base)
try self.writer.put(", #")
try self.writer.put_u64(offset)
try self.writer.put("]")
## A load from a promoted slot: the register, extended as the load would extend from
## memory, since a narrow store leaves the register's upper bits as they were.
func load_promoted(op: ir.Op, k: u32, d: str) -> !:
let code = (usize)self.fr.slot_regs.get((usize)k - 1)
let xs = x_names[code]
let ws = w_names[code]
match op:
.loadhf:
if d != h_names[code]:
try self.fmove(d, h_names[code])
.loadl:
if d != xs:
try self.writer.line4("mov ", d, ", ", xs)
.loaduw:
if d != ws:
try self.writer.line4("mov ", d, ", ", ws)
.loadsw: try self.writer.line4("sxtw ", d, ", ", ws)
.loadub: try self.writer.line5("and ", d, ", ", ws, ", #0xff")
.loadsb: try self.writer.line4("sxtb ", d, ", ", ws)
.loaduh: try self.writer.line5("and ", d, ", ", ws, ", #0xffff")
.loadsh: try self.writer.line4("sxth ", d, ", ", ws)
.loads:
if d != ws:
try self.writer.line4("fmov ", d, ", ", ws)
_:
if d != xs:
try self.writer.line4("fmov ", d, ", ", xs)
## A store into a promoted slot: the value's register `v` moves in.
func store_promoted(op: ir.Op, k: u32, v: str) -> !:
let code = (usize)self.fr.slot_regs.get((usize)k - 1)
match op:
.storel:
if v != x_names[code]:
try self.writer.line4("mov ", x_names[code], ", ", v)
.stores:
if v != w_names[code]:
try self.writer.line4("fmov ", w_names[code], ", ", v)
.storehf:
if v != h_names[code]:
try self.fmove(h_names[code], v)
.stored:
if v != x_names[code]:
try self.writer.line4("fmov ", x_names[code], ", ", v)
_:
if v != w_names[code]:
try self.writer.line4("mov ", w_names[code], ", ", v)
## The prologue: the frame record, the frame below it, the hidden result pointer kept,
## and the callee-saved registers the function uses saved.
func open_frame() -> !:
if self.dw != none:
try self.writer.line(".cfi_startproc")
try self.writer.line("stp x29, x30, [sp, #-16]!")
if self.dw != none:
try self.writer.line(".cfi_def_cfa_offset 16")
try self.writer.line(".cfi_offset 29, -16")
try self.writer.line(".cfi_offset 30, -8")
try self.writer.line("mov x29, sp")
if self.dw != none:
try self.writer.line(".cfi_def_cfa_register 29")
if self.fr.size <= 4095:
try self.writer.line("sub sp, sp, #")
try self.writer.put_u64(self.fr.size)
else:
try self.load_immediate("x16", (i64)self.fr.size)
try self.writer.line("sub sp, sp, x16")
if self.f.has_result and self.f.result_size > 16 and self.hfa(self.f.result_shape) == 0 and not self.f.result_private:
try self.frame_address("x16", self.fr.indirect_home)
try self.writer.line("str x8, [x16]")
try self.save_callee_saved()
if self.f.allocates_frame:
try self.writer.line("mov x28, sp")
self.frame_base = true
func function(index: usize) -> !:
self.index = index
# the prologue's saves are relative to sp at the frame's bottom: nothing of the
# previous function's dynamic area, which its last instruction may have left
# reserved (a statement that never completes), carries over
self.dynamic_shift = 0
if let dw = self.dw:
try dw.entry(&self.writer.out)
self.f = &self.program_unit.functions.items[index]
self.fr.reserved = 0
self.frame_base = false
self.reserve_asm_registers()
if self.f.allocates_frame:
# x28 keeps the frame's bottom (`frame_base`), saved and restored as any
# callee-saved register the function uses
self.fr.reserve(10)
try selection.canonical_labels(self.f)
if not self.f.debug:
try selection.traps_last(self.f)
try selection.sink_addresses(self.f)
try self.fr.plan(self.f)
self.crossing = 0
var entry = try selection.entry_loads(self.f)
defer free(entry)
let straight = try self.straight_params(entry)
defer free(straight)
try self.fr.layout(self.f, 0, self.f.has_result and self.f.result_size > 16 and self.hfa(self.f.result_shape) == 0 and not self.f.result_private, straight)
try self.object_format.function_start(&self.writer.out, self.f)
if self.f.naked:
# the body is asm blocks only (ยง9.8): no frame, no prologue, nothing after
var k: usize = 0
while k < self.f.instrs.count:
try self.instruction(self.f.instrs.get(k))
k += 1
try self.object_format.function_end(&self.writer.out, self.f)
return
try self.open_frame()
var moves = try list.List[incoming.Move].create(8)
defer moves.destroy()
try self.receive_params(entry, &moves)
var uses = try selection.uses(self.f)
defer free(uses)
var received = try self.receive_directly(&moves, uses)
defer free(received)
var zero_upper = try selection.zero_extended(self.f, entry)
defer free(zero_upper)
self.zero_upper = zero_upper
var dynamic = try levels.over(self.f)
defer dynamic.release()
self.dynamic_shift = 0
var heads = try selection.loop_heads(self.f)
defer free(heads)
var tests = try selection.rotations(self.f, heads, uses, true)
defer free(tests)
selection.rotate_heads(self.f, heads, tests)
self.loop_heads = heads
self.head_tests = tests
var roles = try flags.plan(self.f, uses)
defer free(roles)
self.flag_roles = roles
var i: usize = 0
while i < self.f.instrs.count:
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i)
self.dynamic_shift = (u64)dynamic.before[i] * (u64)ir.level_bytes
if received[i]:
i += 1
continue
let fused = try self.fused_at(i, uses, index)
if fused > 0:
i += fused
continue
try self.save_crossing(i)
self.at = i
try self.instruction(self.f.instrs.get(i), selection.label_at(self.f, i + 1))
try self.restore_crossing(i)
if dynamic.settle[i]:
# edges bring different levels: sp goes below the most of them
try self.reserve_levels(dynamic.before[i + 1])
i += 1
self.loop_heads = heads[0..<0]
self.head_tests = tests[0..<0]
self.flag_roles = roles[0..<0]
try self.finish_cold()
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i)
if self.f.debug or self.f.symbols:
try self.writer.put("\nLdw_end_")
try self.writer.put_u64((u64)index)
try self.writer.put(":\n")
try self.writer.line(".cfi_endproc")
try dw.function(self.f, &self.fr, index)
try self.writer.put("\n")
try self.object_format.function_end(&self.writer.out, self.f)
if self.f.debug:
try assembly.describe_frame(&self.descriptors, self.symbol_prefix, self.object_format.text_label, self.program_unit, self.f, &self.fr)
## Emit the instructions from `i` that select as one, when they do (a comparison and
## its branch, an address and its access, a shift and its add): how many they were, or
## 0 when the one at `i` selects alone.
func fused_at(i: usize, uses: u32[], index: usize) -> usize!:
if selection.branch(self.f, i, uses, true):
try self.branch_pair(i, index)
return 2
if let fused = selection.offset_access(self.f, i, uses):
let (base, offset) = fused
let (access, widened) = self.access_at(i + 1, uses)
if not self.fr.is_rematerialised(self.f.instrs.get(i).dst) and offset_reaches(offset, frame.access_width(access.op)):
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
try self.offset_access(access, base, offset)
return 2 + widened
if let taken = try self.scaled_access_at(i, uses, index):
return taken
if let taken = try self.summed_access_at(i, uses, index):
return taken
if let load = selection.widened_load(self.f, i, uses):
try self.instruction(load, selection.label_at(self.f, i + 2))
return 2
if let fused = selection.shifted_add(self.f, i, uses):
if not self.fr.is_rematerialised(self.f.instrs.get(i).dst) and not self.fr.is_rematerialised(self.f.instrs.get(i + 1).dst):
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
try self.shifted_add(self.f.instrs.get(i + 1), fused)
return 2
return 0
## The load or store at `at` a fused address reaches, and 1 when it is a signed word's
## load whose widening after it it does as well (`selection.widened_load`: `ldrsw`
## already extends), else 0.
func access_at(at: usize, uses: u32[]) -> (ir.Instr, usize):
if let load = selection.widened_load(self.f, at, uses):
return (load, 1)
return (self.f.instrs.get(at), 0)
## The comparison at `i` and the branch on it after, as one conditional branch.
func branch_pair(i: usize, index: usize) -> !:
if let dw = self.dw:
# the fused branch's own line and labels, ahead of the pair
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
try self.fused_branch(i, selection.label_at(self.f, i + 2))
## The comparison at `i` and the branch on it after, with `next` as the fall-through:
## the pair in place, or a rotated loop's head test at a jump to that head
## (`selection.rotations`).
func fused_branch(i: usize, next: u32) -> !:
let branch = self.f.instrs.get(i + 1)
var comparison = self.f.instrs.get(i)
var negate = false
if next == branch.extra:
if ir.is_float_class(comparison.cls):
negate = true
else:
comparison.op = ir.negated(comparison.op)
try self.comparison(comparison, branch, next, negate)
## Store the incoming arguments into their slots.
## By slot, the parameters `receive_params` moves straight from their argument registers
## into their entry loads' results (`incoming`): their slots are never written, so they
## need no home. It walks the registers as `receive_params` does, for a function whose
## parameters are all scalars; with an aggregate among them, every slot keeps its home.
func straight_params(entry: u32[]) -> bool[]!:
var straight = try new bool[self.f.slots.count + 1] ---
for k in 0..<straight.length:
straight[k] = false
for p in self.f.params.span():
if p.size != 0:
return straight
var next_int: u32 = 0
var next_float: u32 = 0
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
let moved = self.fr.slot_regs.get((usize)p.slot) == 0 and entry[i] != 0 and p.cls != ir.Class.h
if ir.is_float_class(p.cls):
if next_float < 8:
straight[(usize)p.slot] = moved
next_float += 1
elif next_int < 8:
straight[(usize)p.slot] = moved
next_int += 1
return straight
## Store each parameter where the body reads it: a scalar into its promoted register or
## its slot (or, `straight`, left in its argument register for `receive_directly`), an
## aggregate into its slot, from registers or the caller's stack as AAPCS64 placed it.
func receive_params(entry: u32[], moves: list.List[incoming.Move]*) -> !:
var at = Arguments(next_int = 0, next_float = 0, stack_offset = 16)
# an argument copied by memcpy (past 128 bytes) would clobber the ones left in
# their registers for `receive_directly`
var direct = true
for p in self.f.params.span():
if p.size > 128 and self.hfa(p.shape) == 0:
direct = false
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
let home = self.slot_home(p.slot)
if p.size == 0:
try self.receive_scalar(p, home, direct and entry[i] != 0, entry[i], moves, &at)
elif self.hfa(p.shape) > 0 and at.next_float + self.hfa(p.shape) <= 8:
try self.receive_hfa_registers(p, home, &at)
elif self.hfa(p.shape) > 0:
try self.receive_hfa_stack(p, home, &at)
elif p.size <= 16:
try self.receive_small(p, home, &at)
else:
try self.receive_reference(home, &at)
try self.copy_references()
## A scalar parameter: into its promoted register, else its slot; `straight` when it
## may stay in its argument register for the parallel copy (`moves`) at its entry load.
func receive_scalar(p: ir.Param, home: u64, may_stay: bool, entry_load: u32, moves: list.List[incoming.Move]*, at: Arguments*) -> !:
# a scalar; a promoted slot takes the value in its register
let code = (usize)self.fr.slot_regs.get((usize)p.slot)
let straight = may_stay and code == 0 and p.cls != ir.Class.h
if ir.is_float_class(p.cls):
if at.next_float < 8 and straight:
try moves.push(incoming.Move(at = entry_load - 1, argument = u8(at.next_float), float = true))
at.next_float += 1
elif at.next_float < 8:
if code != 0:
try self.writer.line("fmov ")
try self.writer.put(class_name((u8)code, p.cls))
try self.writer.put(", ")
try self.writer.put(float_letter(p.cls))
else:
try self.frame_address("x16", home)
try self.writer.line("str ")
try self.writer.put(float_letter(p.cls))
try self.writer.put_u64((u64)at.next_float)
if code == 0:
try self.writer.put(", [x16]")
at.next_float += 1
else:
let width = scalar_stack_width(p.cls, p.scalar_width)
let slot = self.stack_slot(width)
at.stack_offset = arm_round_up(at.stack_offset, (u64)slot)
try self.writer.line(scalar_stack_load(width, false))
try self.writer.put(" [x29, #")
try self.writer.put_u64(at.stack_offset)
try self.writer.put("]")
if code != 0:
try self.writer.line3("fmov ", x_names[code], ", x9")
else:
try self.frame_address("x16", home)
try self.writer.line("str x9, [x16]")
at.stack_offset += (u64)slot
elif at.next_int < 8 and straight:
try moves.push(incoming.Move(at = entry_load - 1, argument = u8(at.next_int), float = false))
at.next_int += 1
elif at.next_int < 8:
if code != 0:
try self.writer.line("mov ")
try self.writer.put(w_names[code] if p.cls == ir.Class.w else x_names[code])
try self.writer.put(", ")
try self.writer.put("w" if p.cls == ir.Class.w else "x")
try self.writer.put_u64((u64)at.next_int)
else:
try self.frame_address("x16", home)
try self.writer.line("str w" if p.cls == ir.Class.w else "str x")
try self.writer.put_u64((u64)at.next_int)
try self.writer.put(", [x16]")
at.next_int += 1
else:
let width = scalar_stack_width(p.cls, p.scalar_width)
let slot = self.stack_slot(width)
at.stack_offset = arm_round_up(at.stack_offset, (u64)slot)
try self.writer.line(scalar_stack_load(width, p.scalar_signed))
try self.writer.put(" [x29, #")
try self.writer.put_u64(at.stack_offset)
try self.writer.put("]")
if code != 0:
try self.writer.line3("mov ", w_names[code] if p.cls == ir.Class.w else x_names[code], ", w9" if p.cls == ir.Class.w else ", x9")
else:
try self.frame_address("x16", home)
try self.writer.line("str w9, [x16]" if p.cls == ir.Class.w else "str x9, [x16]")
at.stack_offset += (u64)slot
## A homogeneous float aggregate in d or s registers, into its slot.
func receive_hfa_registers(p: ir.Param, home: u64, at: Arguments*) -> !:
# a homogeneous float aggregate in d or s registers
let each = (u64)p.size // (u64)self.hfa(p.shape)
var k: u32 = 0
while k < self.hfa(p.shape):
try self.frame_address("x16", home)
try self.writer.line("str ")
try self.writer.put(float_letter_of(each))
try self.writer.put_u64((u64)(at.next_float + k))
try self.writer.put(", [x16, #")
try self.writer.put_u64((u64)k * each)
try self.writer.put("]")
k += 1
at.next_float += self.hfa(p.shape)
## A homogeneous float aggregate past the registers, from the caller's stack.
func receive_hfa_stack(p: ir.Param, home: u64, at: Arguments*) -> !:
# An HFA that cannot fit stays an aggregate on the stack. It
# never falls back to integer registers or an indirect pointer,
# and later floating arguments must spill as well (AAPCS64 C.3).
at.next_float = 8
at.stack_offset = arm_round_up(at.stack_offset, self.stack_boundary((u64)p.align))
let each = (u64)self.program_unit.leaves_of(p.shape)[0].size
var k: u32 = 0
while k < self.hfa(p.shape):
try self.writer.line("ldr x9, [x29, #" if each == 8 else ("ldr w9, [x29, #" if each == 4 else "ldrh w9, [x29, #"))
try self.writer.put_u64(at.stack_offset + (u64)k * each)
try self.writer.put("]")
try self.frame_address("x16", home)
try self.writer.line("str x9, [x16, #" if each == 8 else ("str w9, [x16, #" if each == 4 else "strh w9, [x16, #"))
try self.writer.put_u64((u64)k * each)
try self.writer.put("]")
k += 1
at.stack_offset += (u64)p.size
## An aggregate of at most 16 bytes, from one or two registers or the caller's stack.
func receive_small(p: ir.Param, home: u64, at: Arguments*) -> !:
let words = (p.size + 7) // 8
if at.next_int + words <= 8:
try self.frame_address("x16", home)
try self.writer.line("str x")
try self.writer.put_u64((u64)at.next_int)
try self.writer.put(", [x16]")
if words == 2:
try self.writer.line("str x")
try self.writer.put_u64((u64)at.next_int + 1)
try self.writer.put(", [x16, #8]")
at.next_int += words
else:
at.next_int = 8
at.stack_offset = arm_round_up(at.stack_offset, 8)
var k: u32 = 0
while k < words:
try self.writer.line("ldr x9, [x29, #")
try self.writer.put_u64(at.stack_offset)
try self.writer.put("]")
try self.frame_address("x16", home)
try self.writer.line("str x9, [x16, #")
try self.writer.put_u64((u64)k * 8)
try self.writer.put("]")
at.stack_offset += 8
k += 1
## A larger aggregate, passed by reference: its address is kept in the slot for now; the
## bytes are copied in by `copy_references`, after every register is safely stored.
func receive_reference(home: u64, at: Arguments*) -> !:
if at.next_int < 8:
try self.frame_address("x16", home)
try self.writer.line("str x")
try self.writer.put_u64((u64)at.next_int)
try self.writer.put(", [x16]")
at.next_int += 1
else:
at.stack_offset = arm_round_up(at.stack_offset, 8)
try self.writer.line("ldr x9, [x29, #")
try self.writer.put_u64(at.stack_offset)
try self.writer.put("]")
try self.frame_address("x16", home)
try self.writer.line("str x9, [x16]")
at.stack_offset += 8
## Copy each by-reference aggregate's bytes into its slot, over the address that
## `receive_reference` kept there.
func copy_references() -> !:
var i: usize = 0
while i < self.f.params.count:
let p = self.f.params.get(i)
if p.size > 16 and self.hfa(p.shape) == 0:
let home = self.slot_home(p.slot)
if p.size <= 128:
# through scratch: arguments may still wait in x0-x7 (`receive_directly`)
try self.frame_address("x9", home)
try self.writer.line("ldr x10, [x9]")
try self.inline_copy("x10", "x9", (u64)p.size)
else:
try self.frame_address("x0", home)
try self.writer.line("ldr x1, [x0]")
try self.load_immediate("x2", (i64)p.size)
try self.writer.line("bl ")
try self.symbol("memcpy")
i += 1
## The parameters `receive_params` left in their argument registers, moved into their
## entry loads' results as one parallel copy (`incoming`); answers which instructions
## that leaves with nothing to do. x16 parks an argument to break a cycle; x17 carries
## a value to a frame home, all of which are written before any argument is parked.
func receive_directly(moves: list.List[incoming.Move]*, uses: const u32[]) -> bool[]!:
var received = try new bool[self.f.instrs.count + 1] ---
for k in 0..<received.length:
received[k] = false
var sources = try new u8[moves.count + 1] ---
defer free(sources)
var destinations = try new u8[moves.count + 1] ---
defer free(destinations)
for k in 0..<moves.count:
let m = moves.get(k)
received[(usize)m.at] = true
sources[k] = incoming.no_argument
destinations[k] = incoming.no_argument
let n = self.f.instrs.get((usize)m.at)
if (usize)n.dst < uses.length and uses[(usize)n.dst] == 0:
# an unread parameter: its register may be another's, which a copy of this
# one into it would overwrite
continue
if m.float:
# no float argument register is ever a temporary's
let arrived = d_arguments[(usize)m.argument] if n.cls == ir.Class.d else s_arguments[(usize)m.argument]
let code = self.fr.temp_regs.get((usize)n.dst - 1)
if code != 0:
try self.fmove(class_name(code, n.cls), arrived)
else:
try self.frame_store(arrived, self.temp_home(n.dst))
continue
sources[k] = m.argument
let code = self.fr.temp_regs.get((usize)n.dst - 1)
if code >= 23 and code <= 30:
destinations[k] = u8(30 - code)
var steps = try incoming.sequence(sources[..<moves.count], destinations[..<moves.count])
defer steps.destroy()
for step in steps.span():
let m = moves.get((usize)step.index)
if m.float or sources[(usize)step.index] == incoming.no_argument:
continue
if step.park:
try self.writer.line3("mov x16, ", argument_names[(usize)m.argument], "")
continue
let x = "x16" if step.parked else argument_names[(usize)m.argument]
let w = "w16" if step.parked else w_arguments[(usize)m.argument]
try self.receive_one(self.f.instrs.get((usize)m.at), x, w)
return received
## Entry load `n`'s result from the integer register `x` (`w` its narrow view): its
## argument's, or x16; extended as the load from the parameter's slot would have.
func receive_one(n: ir.Instr, x: str, w: str) -> !:
let code = self.fr.temp_regs.get((usize)n.dst - 1)
let d = class_name(code, n.cls) if code != 0 else ("x17" if n.cls == ir.Class.l else "w17")
match n.op:
.loadl:
if d != x:
try self.writer.line4("mov ", d, ", ", x)
.loaduw:
try self.writer.line4("mov ", d, ", ", w)
.loadsw:
try self.writer.line4("sxtw " if n.cls == ir.Class.l else "mov ", d, ", ", w)
.loaduh:
try self.writer.line4("uxth ", narrow_name(d), ", ", w)
.loadsh:
try self.writer.line4("sxth ", d, ", ", w)
.loadub:
try self.writer.line4("uxtb ", narrow_name(d), ", ", w)
_:
try self.writer.line4("sxtb ", d, ", ", w)
if code == 0:
try self.frame_store(d, self.temp_home(n.dst))
## Where the next incoming argument is: the next integer and float argument registers,
## and the offset of the next one the caller passed on the stack.
struct Arguments:
var next_int: u32
var next_float: u32
var stack_offset: u64