dymokomi/luce-base
Source of release 0.43.0
0.43.0 / src / back / native / x86_64 / calls.lucb
#==============================================================================================
#
# x86_64/calls - Calls
#
# DESCRIPTION:
# Calls under the platform's convention: arguments placed, results received.
#
#==============================================================================================
extend Generator:
# mark: Calls ==============================================================================
## The save area's prediction: the nonvolatile xmm6–xmm15, rsi and rdi the allocator
## handed out (codes 6–15, 20 and 21, `registers_of`), each the bit of its place in
## `windows_nonvolatile`; the text may name more (`settle_windows_saves`).
func allocated_saves() -> u16:
var plan: u16 = 0
for code in self.fr.temp_regs.span():
if code >= 6 and code <= 15:
plan = plan | (u16)(1 << (code - 4))
elif code == 20 or code == 21:
plan = plan | (u16)(1 << (21 - code))
return plan
## Windows' nonvolatile registers beyond the allocator's (rdi, rsi and xmm6-xmm15, which
## shared instruction sequences use), saved or restored where `marker` left a line, and
## only those `used` marks, each in its place in the save area (`windows_plan`).
func windows_saved(restore: bool, used: const bool[]) -> !:
for i in 0..<12:
if not used[i]:
continue
let cls = ir.Class.l if i < 2 else ir.Class.v
let offset = save_offset(self.windows_plan, i)
if restore:
try self.frame_load(windows_nonvolatile[i], cls, (u64)offset)
else:
try self.frame_store(windows_nonvolatile[i], cls, (u64)offset)
try self.unwinding.saved_register(&self.writer.out, windows_nonvolatile[i], self.fr.size - (u64)offset, i >= 2)
## Windows x64: the function's text since `start` has a marker line where the prologue
## saves, and one where each exit restores, Windows' further nonvolatile registers; the
## text is scanned for the ones it names, and the markers become the saves and restores
## of those alone, so a function that names none keeps its prologue to the allocator's.
func settle_windows_saves(start: usize) -> !:
if not self.convention.windows or self.f.naked:
return
let length = self.writer.out.used - start
var text = try new u8[length] ---
defer free(text)
memory.copy(text, self.writer.out.bytes()[start..<self.writer.out.used], length)
var used: bool[12] = [false, false, false, false, false, false, false, false, false, false, false, false]
var at: usize = 0
while at < length:
var end = at
while end < length and text[end] != '\n':
end += 1
let line = (str)text[at..<end]
if line != windows_save_marker and line != windows_restore_marker:
for i in 0..<12:
used[i] = used[i] or names_register(line, (usize)i)
at = end + 1
self.windows_used = 0
for i in 0..<12:
if used[i]:
self.windows_used = self.windows_used | (u16)(1 << i)
if (self.windows_used & ~self.windows_plan) != 0:
# the area has no place for one of them: the function is laid out again
return
self.writer.out.used = start
at = 0
while at < length:
var end = at
while end < length and text[end] != '\n':
end += 1
let line = (str)text[at..<end]
if line == windows_save_marker or line == windows_restore_marker:
# each line the saves or restores put starts with its newline: end the last
try self.windows_saved(line == windows_restore_marker, used[0..<12])
try self.writer.out.put_byte('\n')
else:
try self.writer.out.put_bytes(text[at..<end])
if end < length:
try self.writer.out.put_byte('\n')
at = end + 1
## Windows x64: a scalar parameter among the first four that `straight_params` marks goes
## from its argument register straight to its entry load's result, by the parallel copy
## `receive_directly` makes (a result may live in r8, r9, xmm0–xmm3, another argument's
## register); every other parameter is read from where the convention leaves it, the
## shadow area its register was stored to first or the caller's stack, into its slot or
## promoted register.
func receive_windows(entry: u32[], moves: list.List[incoming.Move]*) -> !:
let hidden: usize = 1 if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private) else 0
let straight = try self.straight_params(entry)
defer free(straight)
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
let ordinal = i + hidden
if ordinal >= 4:
continue
if straight[(usize)p.slot]:
try moves.push(incoming.Move(at = entry[i] - 1, argument = u8(ordinal), float = p.size == 0 and ir.is_float_class(p.cls)))
continue
if p.size == 0 and ir.is_float_class(p.cls):
try self.writer.line3("movsd ", float_argument_names[ordinal], ", ")
else:
try self.writer.line3("movq ", windows_arguments[ordinal], ", ")
try self.writer.put_u64(self.fr.size + (u64)(16 + ordinal * 8))
try self.writer.put("(%rbp)")
for i in 0..<self.f.params.count:
let p = self.f.params.get(i)
if straight[(usize)p.slot]:
continue
let home = self.slot_home(p.slot)
let ordinal = i + hidden
if p.size == 0:
try self.stack_param_load("%rax", (u64)(16 + ordinal * 8))
if ir.is_float_class(p.cls):
let width: u64 = 2 if p.cls == ir.Class.h else (4 if p.cls == ir.Class.s else 8)
try self.writer.line2(store_word_mnemonic(width), register_of_width("%rax", width))
try self.writer.put(", ")
try self.frame_operand(home)
else:
let code = self.fr.slot_regs.get((usize)p.slot)
if code != 0:
try self.writer.line4("movq " if p.cls == ir.Class.l else "movl ", "%rax" if p.cls == ir.Class.l else "%eax", ", ", self.class_name((u8)code, p.cls))
else:
try self.frame_store("%rax" if p.cls == ir.Class.l else "%eax", p.cls, home)
elif self.convention.in_memory(p.size, p.shape):
try self.stack_param_load("%r10", (u64)(16 + ordinal * 8))
try self.frame_address("%r11", home)
try self.copy_to_r11("%r10", (u64)p.size)
else:
try self.stack_param_load("%rax", (u64)(16 + ordinal * 8))
try self.frame_address("%r10", home)
try self.store_bytes("%rax", "%r10", 0, (u64)p.size)
func call_windows(n: ir.Instr) -> !:
let info = self.f.calls.get((usize)n.extra)
let hidden = info.has_result and self.result_in_memory(info.result_size, info.result_shape, info.result_private)
let first: u64 = 1 if hidden else 0
let count = (u64)info.args_count + first
let slots = round_up((count if count > 4 else 4) * 8, 16)
var area = slots
for i in 0..<info.args_count:
let a = self.f.args.get((usize)(info.args_start + i))
if self.convention.in_memory(a.size, a.shape):
area += round_up((u64)a.size, 16)
if area >= 4096:
try self.load_immediate("%rax", (i64)area)
try self.writer.line("call ___chkstk_ms")
try self.writer.line("subq $")
try self.writer.put_u64(area)
try self.writer.put(", %rsp")
var copy_offset = slots
for i in 0..<info.args_count:
let a = self.f.args.get((usize)(info.args_start + i))
let offset = ((u64)i + first) * 8
if a.size == 0:
if ir.is_float_class(a.cls):
# read where it lives: an argument register may hold another's value
let v = try self.source(a.value, a.cls, self.reg(a.cls, 0))
try self.writer.line3("movq ", v, ", %rax")
else:
try self.operand("%rax" if a.cls == ir.Class.l else "%eax", a.cls, a.value)
elif self.convention.in_memory(a.size, a.shape):
try self.address_operand("%r10", a.value)
try self.writer.line("leaq ")
try self.writer.put_u64(copy_offset)
try self.writer.put("(%rsp), %r11")
# inline whatever the size: a call of memcpy here would clobber the later
# arguments still living in caller-saved registers
try self.inline_copy("%r10", "%r11", (u64)a.size)
try self.writer.line("leaq ")
try self.writer.put_u64(copy_offset)
try self.writer.put("(%rsp), %rax")
copy_offset += round_up((u64)a.size, 16)
else:
try self.address_operand("%r10", a.value)
try self.load_bytes("%rax", "%r10", 0, (u64)a.size)
try self.writer.line("movq %rax, ")
try self.writer.put_u64(offset)
try self.writer.put("(%rsp)")
if hidden:
try self.frame_address("%rax", self.slot_home(info.result_slot))
try self.writer.line("movq %rax, (%rsp)")
# the callee into r11 while every register still holds what it held: its address
# may live in r8 or r9, which the arguments take
let indirect = n.a.kind == ir.RefKind.temp or n.a.kind == ir.RefKind.constant
if indirect:
try self.operand("%r11", ir.Class.l, n.a)
for i in 0..<(count if count < 4 else 4):
try self.writer.line("movq ")
try self.writer.put_u64(i * 8)
try self.writer.put("(%rsp), ")
try self.writer.put(windows_arguments[i])
if i >= first:
let a = self.f.args.get((usize)((u64)info.args_start + i - first))
if a.size == 0 and ir.is_float_class(a.cls):
try self.writer.line4("movq ", windows_arguments[i], ", ", float_argument_names[i])
if indirect:
try self.writer.line("call *%r11")
else:
try self.writer.line("call ")
try self.symbol(n.a.name)
try self.writer.line("addq $")
try self.writer.put_u64(area)
try self.writer.put(", %rsp")
if not info.has_result or hidden:
return
if info.result_size == 0:
if ir.is_float_class(info.result_cls):
try self.store_temp("%xmm0", info.result_cls, n.dst)
else:
try self.store_temp("%rax" if info.result_cls == ir.Class.l else "%eax", info.result_cls, n.dst)
elif info.result_words:
try self.take_words()
elif info.result_private:
try self.frame_address("%r10", self.slot_home(info.result_slot))
try self.private_words(info.result_size, info.result_shape, true)
else:
try self.frame_address("%r10", self.slot_home(info.result_slot))
try self.store_bytes("%rax", "%r10", 0, (u64)info.result_size)
## The float register arguments (System V) as one parallel move: a float temporary may
## live in another argument's register, xmm0–xmm7, so the scalars living in xmm
## registers move together (`incoming.sequence`), the first float scratch register
## parking one to break a cycle; then the rest, from the frame, from constants, and an
## aggregate's eightbytes through its address, each writing its own register alone. The
## integer arguments come after, from integer registers: none reads a float register.
func float_arguments(info: ir.CallInfo, placements: Placement[]) -> !:
let count = (usize)info.args_count
var sources = try new u8[count + 1] ---
var destinations = try new u8[count + 1] ---
var which = try new usize[count + 1] ---
var moved = try new bool[count + 1] ---
var moves: usize = 0
for i in 0..<count:
let a = self.f.args.get((usize)info.args_start + i)
let p = placements[i]
moved[i] = false
if p.where != 0:
continue
if p.sse == 0 or a.size > 0:
# an integer register argument: its value is in no float register
if a.size == 0 and self.float_register_of(a.value) < 16:
error(failed, "an integer argument whose value lives in a float register")
continue
let held = self.float_register_of(a.value)
if held < 16:
sources[moves] = held
destinations[moves] = (u8)p.first_float
which[moves] = i
moves += 1
moved[i] = true
var steps = try incoming.sequence(sources[..<moves], destinations[..<moves])
defer steps.destroy()
let park = self.reg(ir.Class.v, 0)
for step in steps.span():
let k = (usize)step.index
let held = xmm_names[(usize)sources[k]]
if step.park:
try self.writer.line4("movaps ", held, ", ", park)
continue
let read = park if step.parked else held
let to = float_argument_names[(usize)destinations[k]]
if read != to:
try self.writer.line4("movaps ", read, ", ", to)
for i in 0..<count:
let a = self.f.args.get((usize)info.args_start + i)
let p = placements[i]
if p.where != 0 or p.sse == 0 or moved[i]:
continue
if a.size == 0:
try self.operand(float_argument_names[(usize)p.first_float], a.cls, a.value)
continue
let at = try self.source(a.value, ir.Class.l, "%r10")
var k: u32 = 0
var f = p.first_float
while k < p.words:
if (p.sse & (u8)(1 << k)) != 0:
try self.load_eightbyte(float_argument_names[(usize)f], at, k, a.size, true)
f += 1
k += 1
## The number of the xmm register temporary `r` lives in, or 16 when it is no
## temporary in a float register.
func float_register_of(r: ir.Ref) -> u8:
if r.kind != ir.RefKind.temp or self.fr.is_rematerialised(r.index):
return 16
let code = self.fr.temp_regs.get((usize)r.index - 1)
if code == 0 or not self.fr.float_code(code):
return 16
return xmm_number(self.q_name(code))
## The integer register arguments as one parallel move: an argument may live in an
## argument register another argument takes, so a move waits while its destination is
## still read by a pending one, a cycle breaks through r10 (the frame's address scratch,
## which only the last phase needs), and the arguments read from the frame or made
## from constants come last.
func integer_arguments(info: ir.CallInfo, placements: Placement[]) -> !:
let count = (usize)info.args_count
var pending = try new bool[count + 1] ---
var reads = try new str[count + 1] ---
var i: usize = 0
while i < count:
let a = self.f.args.get((usize)info.args_start + i)
let p = placements[i]
pending[i] = p.where == 0 and (p.sse == 0 or a.size > 0)
reads[i] = ""
if pending[i] and a.value.kind == ir.RefKind.temp:
let code = self.fr.temp_regs.get((usize)a.value.index - 1)
if code != 0 and not self.fr.float_code(code):
reads[i] = self.class_name(code, ir.Class.l)
i += 1
var left = true
while left:
left = false
var moved = false
i = 0
while i < count:
if pending[i] and reads[i].length > 0:
left = true
if not destination_read(placements[i], self.f.args.get((usize)info.args_start + i), pending, reads, i, count):
try self.integer_argument(i, info, placements[i], reads[i])
pending[i] = false
moved = true
i += 1
if left and not moved:
i = 0
while i < count:
if pending[i] and reads[i].length > 0 and argument_index(reads[i]) >= 0:
try self.writer.line3("movq ", reads[i], ", %r10")
reads[i] = "%r10"
break
i += 1
i = 0
while i < count:
if pending[i]:
try self.integer_argument(i, info, placements[i], "")
i += 1
## One integer register argument: from `read` when it lives in a register, else loaded;
## an aggregate's eightbytes come from its address, the one into the address's own
## register last.
func integer_argument(i: usize, info: ir.CallInfo, p: Placement, read: str) -> !:
let a = self.f.args.get((usize)info.args_start + i)
if a.size == 0:
if read.length > 0:
if read != argument_names[(usize)p.first_int]:
try self.writer.line3("movq ", read, ", ")
try self.writer.put(argument_names[(usize)p.first_int])
return
try self.operand(argument_names[(usize)p.first_int] if a.cls == ir.Class.l else narrow_name(argument_names[(usize)p.first_int]), a.cls, a.value)
return
var at = read
if at.length == 0:
at = try self.source(a.value, ir.Class.l, "%r10")
var pass: u32 = 0
while pass < 2:
var k: u32 = 0
var g = p.first_int
while k < p.words:
if (p.sse & (u8)(1 << k)) == 0:
let into = argument_names[(usize)g]
if (into == at) == (pass == 1):
try self.load_eightbyte(into, at, k, a.size, false)
g += 1
k += 1
pass += 1
func call(n: ir.Instr) -> !:
if self.convention.windows:
try self.call_windows(n)
return
let info = self.f.calls.get((usize)n.extra)
let hidden = info.has_result and self.result_in_memory(info.result_size, info.result_shape, info.result_private)
var next_int: u32 = 1 if hidden else 0
var next_float: u32 = 0
var stack: u32 = 0
# one placement per argument, however many: from the build's arena
var placements: Placement[]
if info.args_count > 0:
placements = try new Placement[(usize)info.args_count] ---
var i: u32 = 0
while i < info.args_count:
let a = self.f.args.get((usize)(info.args_start + i))
placements[i] = self.convention.place(a, &next_int, &next_float, &stack)
i += 1
let area = round_up((u64)stack, 16)
if area > 0:
try self.writer.line("subq $")
try self.writer.put_u64(area)
try self.writer.put(", %rsp")
# stack arguments first, while every register is still free
i = 0
while i < info.args_count:
let a = self.f.args.get((usize)(info.args_start + i))
let p = placements[i]
if p.where == 2:
if a.size == 0:
let cls = ir.Class.l if a.cls == ir.Class.w or a.cls == ir.Class.l else a.cls
try self.operand(self.reg(cls, 0), cls, a.value)
try self.writer.line3(move_of(cls), self.reg(cls, 0), ", ")
try self.writer.put_u64((u64)p.stack_offset)
try self.writer.put("(%rsp)")
else:
let at = try self.source(a.value, ir.Class.l, "%r10")
try self.writer.line("leaq ")
try self.writer.put_u64((u64)p.stack_offset)
try self.writer.put("(%rsp), %r11")
# inline whatever the size: a call here would clobber the arguments
# already living in caller-saved registers
try self.inline_copy(at, "%r11", (u64)a.size)
i += 1
# the callee first, into r11, which no argument's placement touches: its home may be
# an argument register (a constant propagated into an indirect call has no symbol)
if n.a.kind == ir.RefKind.temp or n.a.kind == ir.RefKind.constant:
try self.operand("%r11", ir.Class.l, n.a)
# float registers, then integer ones, so loading the latter disturbs nothing
try self.float_arguments(info, placements)
try self.integer_arguments(info, placements)
if hidden:
try self.frame_address("%rdi", self.slot_home(info.result_slot))
if info.variadic:
try self.load_immediate("%eax", (i64)next_float)
if n.a.kind == ir.RefKind.temp or n.a.kind == ir.RefKind.constant:
try self.writer.line("call *%r11")
else:
try self.call_symbol(n.a.name)
if area > 0:
try self.writer.line("addq $")
try self.writer.put_u64(area)
try self.writer.put(", %rsp")
if not info.has_result:
return
if info.result_size == 0:
if ir.is_float_class(info.result_cls):
try self.store_temp("%xmm0", info.result_cls, n.dst)
else:
try self.store_temp("%rax" if info.result_cls == ir.Class.l else "%eax", info.result_cls, n.dst)
return
if hidden:
return
if info.result_words:
try self.take_words()
return
try self.frame_address("%r10", self.slot_home(info.result_slot))
if info.result_private:
try self.private_words(info.result_size, info.result_shape, true)
return
let sse = self.convention.sse_of(info.result_cls, info.result_size, info.result_shape)
try self.result_eightbytes(info.result_size, sse, true)
## Whether a result goes through the hidden pointer: an aggregate the C convention
## passes in memory, unless it takes the program's own (CallInfo.result_private).
func result_in_memory(size: u32, shape: u32, private: bool) -> bool:
return not private and self.convention.in_memory(size, shape)
## The words of a private result of up to four words (CallInfo.result_private) between
## (%r10) and the registers it comes back in, the program's own convention on both
## System V and Windows (all eight are caller-saved on both): a word holding one float
## (`ir.Unit.word_class`) in the next of xmm0-xmm3, any other in the next of rax, rdx,
## rcx and r8; `store` into the memory from the registers, else the reverse. Loaded
## from the last word: an odd-width word takes rax to help (`load_bytes`), so rax's own
## word comes last.
func private_words(size: u32, shape: u32, store: bool) -> !:
let words = (size + 7) // 8
var registers: str[4] = ["", "", "", ""]
var floats: bool[4] = [false, false, false, false]
var ints: usize = 0
var sse: usize = 0
for k in 0..<words:
floats[k] = self.program_unit.word_class(shape, k) != ir.Class.l
registers[k] = xmm_names[sse] if floats[k] else private_result_registers[ints]
sse += 1 if floats[k] else 0
ints += 0 if floats[k] else 1
for j in 0..<words:
let k = j if store else words - 1 - j
if store:
try self.store_eightbyte(registers[k], k, size, floats[k])
else:
try self.load_eightbyte(registers[k], "%r10", k, size, floats[k])
## The eightbytes of a small aggregate result between (%r10) and rax/rdx and xmm0/xmm1,
## `store` into the memory from the registers, else the reverse, the last word first
## as `private_words` loads them.
func result_eightbytes(size: u32, sse: u8, store: bool) -> !:
let words: u32 = 2 if size > 8 else 1
for j in 0..<words:
let k = j if store else words - 1 - j
let float = (sse & (u8)(1 << k)) != 0
# the second word takes the second register of its class only when the first
# word is of that class too
let second = k == 1 and ((sse & 1) != 0) == float
let reg = ("%xmm1" if second else "%xmm0") if float else ("%rdx" if second else "%rax")
if store:
try self.store_eightbyte(reg, k, size, float)
else:
try self.load_eightbyte(reg, "%r10", k, size, float)
## %rsp below `count` levels of the dynamic area under the fixed frame (`ir.Op.reserve`),
## set from %rbp whatever it was; on Windows a call's 32-byte shadow area stays below
## the levels too.
func reserve_levels(count: u32) -> !:
var below = (u64)count * (u64)ir.level_bytes
if self.convention.windows:
below += 32 if count > 0 else 0
# Windows commits the stack a guard page at a time: touch each page on the way
var page: u64 = 4096
while page < below:
try self.writer.line("orq $0, -")
try self.writer.put_u64(page)
try self.writer.put("(%rbp)")
page += 4096
else:
below += self.fr.size
try self.writer.line("leaq -")
try self.writer.put_u64(below)
try self.writer.put("(%rbp), %rsp")
## The words of a result in registers, from rax, rdx, rcx and r8 and from xmm0-xmm3
## into the `word`s right after the call at `self.at`, each word taking the next register
## of its kind: those kept in the frame stored first, then the rest as two parallel
## copies (`incoming`), the integer words' and the float words', since a word may live
## in another's register (r8, xmm0-xmm3); r11 and the first float scratch register
## park one to break a cycle.
func take_words() -> !:
var sources: u8[4] = [0, 0, 0, 0]
var destinations: u8[4] = [0, 0, 0, 0]
var targets: u8[4] = [0, 0, 0, 0]
var dsts: u32[4] = [0, 0, 0, 0]
var count: usize = 0
var float_sources: u8[4] = [0, 0, 0, 0]
var float_destinations: u8[4] = [0, 0, 0, 0]
var float_targets: u8[4] = [0, 0, 0, 0]
var float_count: usize = 0
var ints: u32 = 0
var floats: u32 = 0
var j = self.at + 1
while j < self.f.instrs.count and self.f.instrs.get(j).op == ir.Op.word:
let w = self.f.instrs.get(j)
j += 1
let float = ir.is_float_class(w.cls)
let index = floats if float else ints
floats += 1 if float else 0
ints += 0 if float else 1
if w.dst == 0 or index >= 4:
continue
let code = self.fr.temp_regs.get((usize)w.dst - 1)
if float:
if code == 0:
try self.frame_store(xmm_names[(usize)index], w.cls, self.temp_home(w.dst))
continue
let held = xmm_number(self.q_name(code))
float_sources[float_count] = u8(index)
float_destinations[float_count] = held if held < 4 else incoming.no_argument
float_targets[float_count] = held
float_count += 1
continue
if code == 0:
let arrived = private_result_narrow[(usize)index] if w.cls == ir.Class.w else private_result_registers[(usize)index]
try self.frame_store(arrived, w.cls, self.temp_home(w.dst))
continue
sources[count] = u8(index)
destinations[count] = incoming.no_argument
for r in 0..<private_result_registers.length:
if private_result_registers[r] == self.q_name(code):
destinations[count] = u8(r)
targets[count] = code
dsts[count] = w.dst
count += 1
var steps = try incoming.sequence(sources[..<count], destinations[..<count])
defer steps.destroy()
for step in steps.span():
let k = (usize)step.index
let arrived = private_result_registers[(usize)sources[k]]
if step.park:
try self.writer.line3("movq ", arrived, ", %r11")
continue
# a word of class w moves its low half, which zeroes the upper
if self.f.temp_class(dsts[k]) == ir.Class.w:
let read = "%r11d" if step.parked else private_result_narrow[(usize)sources[k]]
try self.writer.line4("movl ", read, ", ", self.l_name(targets[k]))
continue
let to = self.q_name(targets[k])
let read = "%r11" if step.parked else arrived
if to != read:
try self.writer.line4("movq ", read, ", ", to)
var float_steps = try incoming.sequence(float_sources[..<float_count], float_destinations[..<float_count])
defer float_steps.destroy()
let park = self.reg(ir.Class.v, 0)
for step in float_steps.span():
let k = (usize)step.index
let arrived = xmm_names[(usize)float_sources[k]]
if step.park:
try self.writer.line4("movaps ", arrived, ", ", park)
continue
let read = park if step.parked else arrived
let to = xmm_names[(usize)float_targets[k]]
if to != read:
try self.writer.line4("movaps ", read, ", ", to)
## A `ret` of a result's words, its arguments each into the next register of its kind:
## the integer ones into rax, rdx, rcx and r8, the floats into xmm0-xmm3. Those living
## in registers move first, as one parallel copy for each kind (r8 and xmm0-xmm3 are
## both allocated registers and a result's), r11 and the first float scratch register
## parking one to break a cycle; then the rest, loaded or formed straight into theirs.
func return_words(n: ir.Instr) -> !:
var count: usize = 0
var words: u32[4] = [0, 0, 0, 0]
var float_count: usize = 0
var float_words: u32[4] = [0, 0, 0, 0]
for k in 0..<(usize)n.extra2:
let a = self.f.args.get((usize)n.extra + k)
if ir.is_float_class(a.cls):
if float_count < 4:
float_words[float_count] = u32(k)
float_count += 1
elif count < 4:
words[count] = u32(k)
count += 1
var float_sources: u8[4] = [0, 0, 0, 0]
var float_destinations: u8[4] = [0, 0, 0, 0]
var float_moved: bool[4] = [false, false, false, false]
var moves: usize = 0
var which: usize[4] = [0, 0, 0, 0]
for k in 0..<float_count:
let held = self.float_register_of(self.f.args.get((usize)n.extra + (usize)float_words[k]).value)
if held < 16:
float_sources[moves] = held
float_destinations[moves] = u8(k)
which[moves] = k
float_moved[k] = true
moves += 1
var float_steps = try incoming.sequence(float_sources[..<moves], float_destinations[..<moves])
defer float_steps.destroy()
let park = self.reg(ir.Class.v, 0)
for step in float_steps.span():
let m = (usize)step.index
let held = xmm_names[(usize)float_sources[m]]
if step.park:
try self.writer.line4("movaps ", held, ", ", park)
continue
let read = park if step.parked else held
let to = xmm_names[which[m]]
if read != to:
try self.writer.line4("movaps ", read, ", ", to)
for k in 0..<float_count:
if not float_moved[k]:
let a = self.f.args.get((usize)n.extra + (usize)float_words[k])
try self.operand(xmm_names[k], a.cls, a.value)
var reads: str[4] = ["", "", "", ""]
var pending: bool[4] = [false, false, false, false]
for k in 0..<count:
let v = self.f.args.get((usize)n.extra + (usize)words[k]).value
if v.kind == ir.RefKind.temp and not self.fr.is_rematerialised(v.index):
let code = self.fr.temp_regs.get((usize)v.index - 1)
if code != 0 and not self.fr.float_code(code):
reads[k] = self.q_name(code)
pending[k] = true
var left = true
while left:
left = false
var moved = false
for k in 0..<count:
if not pending[k]:
continue
left = true
let target = private_result_registers[k]
var blocked = false
for other in 0..<count:
if other != k and pending[other] and reads[other] == target:
blocked = true
if not blocked:
if reads[k] != target:
try self.writer.line4("movq ", reads[k], ", ", target)
pending[k] = false
moved = true
if left and not moved:
for k in 0..<count:
if pending[k]:
try self.writer.line3("movq ", reads[k], ", %r11")
reads[k] = "%r11"
break
for k in 0..<count:
if reads[k].length == 0:
try self.operand(private_result_registers[k], ir.Class.l, self.f.args.get((usize)n.extra + (usize)words[k]).value)
func return_value(n: ir.Instr) -> !:
if self.f.has_result and n.extra2 == 0:
if self.f.result_size == 0:
if ir.is_float_class(self.f.result_cls):
try self.operand("%xmm0", self.f.result_cls, n.a)
else:
try self.operand("%rax" if self.f.result_cls == ir.Class.l else "%eax", self.f.result_cls, n.a)
elif self.f.result_private:
try self.address_operand("%r10", n.a)
try self.private_words(self.f.result_size, self.f.result_shape, false)
elif not self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private):
try self.address_operand("%r10", n.a)
let sse = self.convention.sse_of(self.f.result_cls, self.f.result_size, self.f.result_shape)
try self.result_eightbytes(self.f.result_size, sse, false)
else:
try self.address_operand(self.c_argument(1), n.a)
try self.writer.line("movq ")
try self.frame_operand(self.fr.indirect_home)
try self.put2(", ", self.c_argument(0))
if self.f.result_size <= 128:
# a few words: loads and stores, not a call, on every return
try self.inline_copy(self.c_argument(1), self.c_argument(0), (u64)self.f.result_size)
else:
try self.load_immediate(self.c_argument(2), (i64)self.f.result_size)
try self.call_symbol("memcpy")
try self.writer.line("movq ")
try self.frame_operand(self.fr.indirect_home)
try self.writer.put(", %rax")
try self.restore_callee_saved()
try self.unwinding.begin_epilogue(&self.writer.out)
if self.convention.windows:
try self.frame_address("%rsp", 0)
else:
try self.writer.line("movq %rbp, %rsp")
try self.writer.line("popq %rbp")
try self.unwinding.popped_frame_pointer(&self.writer.out)
try self.writer.line("ret")
try self.unwinding.end_epilogue(&self.writer.out)
## The callee-saved registers the function's asm blocks name as operands, `rbx` and
## `r12`-`r15` in any width, and on Windows the nonvolatile xmm6–xmm15: reserved, so no
## value of the function lives in one, and they are saved and restored like the
## allocator's.
func reserve_asm_registers():
let callee = self.fr.registers.ints + self.fr.registers.floats
for n in self.f.instrs.span():
if n.op != ir.Op.set_reg and n.op != ir.Op.get_reg:
continue
let reg = with_percent(self.program_unit.texts.get((usize)n.b.index).bytes)
for c in 1..<callee + 1:
if reg == self.q_name(c) or reg == self.l_name(c) or reg == self.w_name(c) or reg == self.b_name(c):
self.fr.reserve(c)
func save_callee_saved() -> !:
if self.convention.windows:
try self.writer.put("\n")
try self.writer.put(windows_save_marker)
var c: usize = 1
while c <= 5:
if (self.fr.used_mask & (u32)(1 << (u32)(c - 1))) != 0:
try self.frame_store(self.q_name((u8)c), ir.Class.l, self.fr.save_offset[c])
try self.unwinding.saved_register(&self.writer.out, self.q_name((u8)c), self.fr.size - self.fr.save_offset[c], false)
try self.unwinding.saved_cfi_register(&self.writer.out, 3 if c == 1 else (u64)c + 10, self.fr.save_offset[c] + 16)
c += 1
func restore_callee_saved() -> !:
if self.convention.windows:
try self.writer.put("\n")
try self.writer.put(windows_restore_marker)
var c: usize = 1
while c <= 5:
if (self.fr.used_mask & (u32)(1 << (u32)(c - 1))) != 0:
try self.frame_load(self.q_name((u8)c), ir.Class.l, self.fr.save_offset[c])
c += 1
## Whether a pending argument other than `i` reads a register argument `i` writes.
func destination_read(p: Placement, a: ir.Arg, pending: bool[], reads: str[], i: usize, count: usize) -> bool:
let words = integer_words(p, a)
var k: usize = 0
while k < count:
if k != i and pending[k] and reads[k].length > 0:
let read = argument_index(reads[k])
if read >= (i32)p.first_int and read < (i32)p.first_int + (i32)words:
return true
k += 1
return false
## How many integer registers argument `i` writes: one, or an aggregate's integer eightbytes.
func integer_words(p: Placement, a: ir.Arg) -> u32:
if a.size == 0:
return 1
var words: u32 = 0
var k: u32 = 0
while k < p.words:
if (p.sse & (u8)(1 << k)) == 0:
words += 1
k += 1
return words