dymokomi/luce-base
Source of release 0.45.0
0.45.0 / src / back / native / x86_64 / functions.lucb
#==============================================================================================
#
# x86_64/functions - Functions
#
# DESCRIPTION:
# Each function's prologue, parameters received under the convention, body, and
# epilogue.
#
#==============================================================================================
extend Generator:
# mark: Functions ==========================================================================
## Copy `size` bytes from (`origin`) to (`to`) with plain loads and stores: 16 at a
## time through the first float scratch register, then 8, 4, 2, 1 through rax.
## Unaligned access is fine on this target.
func inline_copy(origin: str, to: str, size: u64) -> !:
var done: u64 = 0
let through = self.reg(ir.Class.v, 0)
while size - done >= 16:
try self.based_move("movups ", done, origin, ", ")
try self.writer.put(through)
try self.writer.line3("movups ", through, ", ")
try self.writer.put_u64(done)
try self.writer.put("(")
try self.put2(to, ")")
done += 16
let widths: u64[4] = [8, 4, 2, 1]
let loads: str[4] = ["movq ", "movl ", "movzwl ", "movzbl "]
let load_regs: str[4] = [", %rax", ", %eax", ", %eax", ", %eax"]
let stores: str[4] = ["movq %rax, ", "movl %eax, ", "movw %ax, ", "movb %al, "]
var k: usize = 0
while k < 4:
if size - done >= widths[k]:
try self.based_move(loads[k], done, origin, load_regs[k])
try self.based_move(stores[k], done, to, "")
done += widths[k]
k += 1
## Zero `size` bytes at (`to`) the same way.
func inline_zero(to: str, size: u64) -> !:
var done: u64 = 0
let through = self.reg(ir.Class.v, 0)
if size >= 16:
try self.writer.line4("pxor ", through, ", ", through)
while size - done >= 16:
try self.writer.line3("movups ", through, ", ")
try self.writer.put_u64(done)
try self.writer.put("(")
try self.put2(to, ")")
done += 16
let widths: u64[4] = [8, 4, 2, 1]
let stores: str[4] = ["movq $0, ", "movl $0, ", "movw $0, ", "movb $0, "]
var k: usize = 0
while k < 4:
if size - done >= widths[k]:
try self.based_move(stores[k], done, to, "")
done += widths[k]
k += 1
## `mnemonic offset(base)rest`.
func based_move(mnemonic: str, offset: u64, base: str, rest: str) -> !:
try self.writer.line(mnemonic)
try self.writer.put_u64(offset)
try self.writer.put("(")
try self.writer.put(base)
try self.writer.put(")")
try self.writer.put(rest)
## `mnemonic` + offset + `rest`: one line of a copy.
func offset_move(mnemonic: str, offset: u64, rest: str) -> !:
try self.writer.line(mnemonic)
try self.writer.put_u64(offset)
try self.writer.put(rest)
## Whether the function needs the third float scratch register (`reg(cls, 2)`): a fused
## multiply-add, a checked vector sum or difference, or an aggregate passed or returned
## whose last eightbyte may go between memory and an SSE register in part
## (`load_eightbyte`).
func third_scratch() -> bool:
if self.f.has_result and partial_eightbyte(self.f.result_size):
return true
for n in self.f.instrs.span():
if n.op == ir.Op.fma or n.op == ir.Op.vaddo or n.op == ir.Op.vsubo:
return true
if n.op == ir.Op.call:
let info = self.f.calls.get((usize)n.extra)
if info.has_result and partial_eightbyte(info.result_size):
return true
for k in 0..<info.args_count:
if partial_eightbyte(self.f.args.get((usize)(info.args_start + k)).size):
return true
return false
## The instructions that overwrite rdx or rcx, temporaries' registers, themselves
## (`frame.Frame.fixed`): a division (rdx:rax), an unsigned checked multiply by other
## than a constant (rdx:rax), a shift by a variable count (cl), an atomic operation
## retried in a loop (rdx), and below x86-64-v3 a half's conversion through the unit's
## routines, which use both. No temporary lives in those across them; their operands
## and results may, and each sequence moves them out of the way.
func fixed_clobbers() -> !:
self.fr.fixed.count = 0
let rdx = (u64)1 << (u64)(fixed_code(self.convention.windows, false) - 1)
let rcx = (u64)1 << (u64)(fixed_code(self.convention.windows, true) - 1)
for i in 0..<self.f.instrs.count:
let n = self.f.instrs.get(i)
var codes: u64 = 0
match n.op:
.sdiv, .udiv, .srem, .urem:
if is_int_class(n.cls):
codes = rdx
.shl, .lshr, .ashr:
if is_int_class(n.cls) and not fits_immediate(n.b):
codes = rcx
.mulo:
if n.extra != 1 and not constant_factor(n.b):
codes = rdx
.atomic_or, .atomic_xor, .atomic_and_not, .atomic_max, .atomic_min:
codes = rdx
_:
_ = 0
if self.level < 3 and (n.cls == ir.Class.h or n.op == ir.Op.exth):
codes = codes | rdx | rcx
if codes != 0:
try self.fr.fixed.push(regalloc.Clobber(at = u32(i), codes = codes))
## A load from a promoted slot: the register, extended as the load would extend from
## memory, since a narrow store leaves the register's upper bits as they were.
func load_promoted(op: ir.Op, k: u32, d: str) -> !:
let code = self.fr.slot_regs.get((usize)k - 1)
let q = self.q_name(code)
let l = self.l_name(code)
match op:
.loadl:
if d != q:
try self.writer.line4("movq ", q, ", ", d)
.loaduw:
if d != l:
try self.writer.line4("movl ", l, ", ", d)
.loadsw: try self.writer.line4("movslq ", l, ", ", d)
.loadub: try self.writer.line4("movzbl ", self.b_name(code), ", ", d)
.loadsb: try self.writer.line4("movsbl ", self.b_name(code), ", ", d)
.loaduh: try self.writer.line4("movzwl ", self.w_name(code), ", ", d)
.loadsh: try self.writer.line4("movswl ", self.w_name(code), ", ", d)
_:
error(failed, "a float load from a promoted slot")
## A store into a promoted slot: the value's register `v` moves in.
func store_promoted(op: ir.Op, k: u32, v: str) -> !:
let code = self.fr.slot_regs.get((usize)k - 1)
match op:
.storel:
if v != self.q_name(code):
try self.writer.line4("movq ", v, ", ", self.q_name(code))
.storew:
if v != self.l_name(code):
try self.writer.line4("movl ", v, ", ", self.l_name(code))
.storeh: try self.writer.line4("movw ", word_name(v), ", ", self.w_name(code))
.storeb: try self.writer.line4("movb ", byte_name(v), ", ", self.b_name(code))
_:
error(failed, "a float store into a promoted slot")
## Emit the instructions from `i` that select as one, when they do (a comparison and
## its branch, an address and its access): how many they were, or 0 when the one at `i`
## selects alone.
func fused_at(i: usize, uses: u32[], index: usize) -> usize!:
if selection.branch(self.f, i, uses, true):
try self.branch_pair(i, index)
return 2
if selection.compared_select(self.f, i, uses):
try self.mark_at(index, i + 1)
try self.select_on(self.f.instrs.get(i), self.f.instrs.get(i + 1))
return 2
if let fused = selection.offset_access(self.f, i, uses):
let (base, offset) = fused
if not self.fr.is_rematerialised(self.f.instrs.get(i).dst):
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
let (access, widened) = self.access_at(i + 1, uses)
try self.mark_widening(index, i + 1, widened)
try self.offset_access(access, base, offset)
return 2 + widened
if let scaled = selection.scaled_access(self.f, i, uses):
let (base, scaled_index, shift) = scaled
if not self.fr.is_rematerialised(self.f.instrs.get(i).dst) and not self.fr.is_rematerialised(self.f.instrs.get(i + 1).dst):
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 2)
let (access, widened) = self.access_at(i + 2, uses)
try self.mark_widening(index, i + 2, widened)
try self.indexed_access(access, base, scaled_index, shift)
return 3 + widened
if let summed = selection.summed_access(self.f, i, uses):
let (base, offset) = summed
if not self.fr.is_rematerialised(self.f.instrs.get(i).dst):
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
let (access, widened) = self.access_at(i + 1, uses)
try self.mark_widening(index, i + 1, widened)
try self.indexed_access(access, base, offset, 0)
return 2 + widened
if let load = selection.widened_load(self.f, i, uses):
try self.mark_at(index, i + 1)
try self.instruction(load, selection.label_at(self.f, i + 2))
return 2
return 0
## The debug labels and lines of instruction `at`, which a fused selection consumes: a
## scope may begin or end there, so its label must exist.
func mark_at(index: usize, at: usize) -> !:
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, at)
## The same for the widening after the access at `at`, when the access took it as well.
func mark_widening(index: usize, at: usize, widened: usize) -> !:
if widened != 0:
try self.mark_at(index, at + 1)
## The load or store at `at` a fused address reaches, and 1 when it is a signed word's
## load whose widening after it it does as well (`selection.widened_load`), else 0.
func access_at(at: usize, uses: u32[]) -> (ir.Instr, usize):
if let load = selection.widened_load(self.f, at, uses):
return (load, 1)
return (self.f.instrs.get(at), 0)
## The comparison at `i` and the branch on it after, as one conditional jump.
func branch_pair(i: usize, index: usize) -> !:
if let dw = self.dw:
# the fused branch's own line and labels, ahead of the pair
try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
try self.fused_branch(i, selection.label_at(self.f, i + 2))
## The comparison at `i` and the branch on it after, with `next` as the fall-through:
## the pair in place, or a rotated loop's head test at a jump to that head
## (`selection.rotations`).
func fused_branch(i: usize, next: u32) -> !:
let branch = self.f.instrs.get(i + 1)
var comparison = self.f.instrs.get(i)
if ir.is_float_class(comparison.cls):
# a float comparison's negation is no comparison of its own: unordered operands
# fail both
try self.float_branch(comparison, branch, next)
return
if next == branch.extra:
comparison.op = ir.negated(comparison.op)
try self.comparison(comparison, branch, next)
## `.weak` for a weak symbol (§9.8); ELF has no directive for `used`, which only matters
## to a linker that discards sections, so it is left to the section's own flags.
func function(index: usize) -> !:
self.index = index
if let dw = self.dw:
try dw.entry(&self.writer.out)
self.f = &self.program_unit.functions.items[index]
let start = self.writer.out.used
try self.function_text(index, none)
# Windows: the save area holds just the nonvolatile registers the text names; when
# the allocator's prediction of them missed, the function is laid out again with
# those (its registers do not depend on the area, so the second text names the same)
if self.convention.windows and not self.f.naked and self.dw == none and self.windows_used != self.windows_plan:
self.writer.out.used = start
try self.function_text(index, self.windows_used)
if self.windows_used != self.windows_plan:
error(failed, "the Windows nonvolatile registers a function names changed when it was laid out again")
## Function `index`'s text; on Windows with its save area holding `plan`, or the
## registers the allocator gave it (a debug build's, all of them) when none.
func function_text(index: usize, plan: u16?) -> !:
self.fr.reserved = 0
self.pool.count = 0
if self.third_scratch():
self.fr.reserve(third_scratch_code(self.convention.windows))
self.reserve_asm_registers()
try layout.arrange(self.f, false)
try selection.sink_addresses(self.f)
try self.fixed_clobbers()
try self.fr.plan(self.f)
var threaded = try layout.thread_after_allocation(self.f, &self.fr)
defer threaded.restore(self.f)
self.windows_plan = 0
if self.convention.windows:
self.windows_plan = plan else (4095 if self.dw != none else self.allocated_saves())
self.crossing = 0
var entry = try selection.entry_loads(self.f)
defer free(entry)
let straight = try self.straight_params(entry)
defer free(straight)
try self.fr.layout(self.f, save_area(self.windows_plan), self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private), straight)
let text_start = self.writer.out.used
try self.object_format.function_start(&self.writer.out, self.f)
if self.f.naked:
# the body is asm blocks only (§9.8): no frame, no prologue, nothing after, not
# even the `ret` the lowerer ends every function with
var k: usize = 0
while k < self.f.instrs.count:
self.at = k
if self.f.instrs.get(k).op == ir.Op.asm_:
try self.instruction(self.f.instrs.get(k))
k += 1
try self.object_format.function_end(&self.writer.out, self.f)
return
try self.unwinding.begin(&self.writer.out, self.symbol_prefix, self.f.name)
try self.writer.line("pushq %rbp")
try self.unwinding.pushed_frame_pointer(&self.writer.out)
if not self.convention.windows:
try self.writer.line("movq %rsp, %rbp")
try self.unwinding.cfa_frame_pointer(&self.writer.out)
if self.convention.windows and self.fr.size >= 4096:
try self.load_immediate("%rax", (i64)self.fr.size)
try self.writer.line("call ___chkstk_ms")
if self.fr.size > 0:
try self.writer.line("subq $")
try self.writer.put_u64(self.fr.size)
try self.writer.put(", %rsp")
try self.unwinding.allocated_stack(&self.writer.out, self.fr.size)
if self.convention.windows:
try self.writer.line("movq %rsp, %rbp")
try self.unwinding.established_frame(&self.writer.out, self.fr.size)
if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private):
try self.writer.line("movq %rcx, " if self.convention.windows else "movq %rdi, ")
try self.frame_operand(self.fr.indirect_home)
try self.save_callee_saved()
try self.unwinding.prologue_complete(&self.writer.out)
var moves = try list.List[incoming.Move].create(8)
defer moves.destroy()
try self.receive_params(entry, &moves)
var uses = try selection.uses(self.f)
defer free(uses)
var received = try self.receive_directly(&moves, uses)
defer free(received)
var dynamic = try levels.over(self.f)
defer dynamic.release()
var heads = try selection.loop_heads(self.f)
defer free(heads)
var tests = try selection.rotations(self.f, heads, uses, false)
defer free(tests)
selection.rotate_heads(self.f, heads, tests)
self.loop_heads = heads
self.head_tests = tests
var i: usize = 0
while i < self.f.instrs.count:
self.at = i
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, i)
if received[i]:
i += 1
continue
let fused = try self.fused_at(i, uses, index)
if fused > 0:
i += fused
continue
try self.save_crossing(i)
try self.instruction(self.f.instrs.get(i), selection.label_at(self.f, i + 1))
try self.restore_crossing(i)
if dynamic.settle[i]:
# edges bring different levels: %rsp goes below the most of them
try self.reserve_levels(dynamic.before[i + 1])
i += 1
self.loop_heads = heads[0..<0]
self.head_tests = tests[0..<0]
try self.finish_function(index, i, text_start)
## After function `index`'s last instruction (`end`): its debug information, unwinding
## and Windows saves (its text began at `text_start`), then out of its section.
func finish_function(index: usize, end: usize, text_start: usize) -> !:
try self.finish_cold()
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, end)
if self.f.debug or self.f.symbols:
try self.writer.put("\nLdw_end_")
try self.writer.put_u64((u64)index)
try self.writer.put(":\n")
try self.unwinding.cfi_complete(&self.writer.out)
try dw.function(self.f, &self.fr, index)
try self.unwinding.complete(&self.writer.out)
try self.writer.put("\n")
try self.settle_windows_saves(text_start)
if self.f.debug:
try assembly.describe_frame(&self.descriptors, self.symbol_prefix, ".Ltext_", self.program_unit, self.f, &self.fr)
try self.object_format.function_end(&self.writer.out, self.f)
try self.finish_pool()