dymokomi/luce-base
Source of release 0.45.0
0.45.0 / src / back / native / arm64 / instructions.lucb
#==============================================================================================
#
# arm64/instructions - Instructions
#
# DESCRIPTION:
# One IR instruction at a time: moves, arithmetic, compares, loads and stores, and
# the checked operations that trap.
#
#==============================================================================================
extend Generator:
# mark: Instructions =======================================================================
func instruction(n: ir.Instr, next_label: u32 = 0) -> !:
if n.dst != 0 and self.fr.is_rematerialised(n.dst):
# a frame address is formed where it is used
return
match n.op:
.dbg:
_ = 0
.label:
if (usize)n.extra < self.loop_heads.length and self.loop_heads[(usize)n.extra]:
# a loop starts on a sixteen-byte boundary
try self.writer.put("\n .p2align 4")
try self.writer.put("\n")
try self.label_name(n.extra)
try self.writer.put(":")
.jmp:
if n.extra == next_label:
return
if (usize)n.extra < self.head_tests.length and self.head_tests[(usize)n.extra] != 0:
# the head's own test, falling through to what follows the jump
try self.fused_branch((usize)self.head_tests[(usize)n.extra] - 1, next_label)
return
try self.writer.line("b ")
try self.label_name(n.extra)
.jnz:
let c = try self.source(n.a, n.cls, "x9" if n.cls == ir.Class.l else "w9")
# into whichever target comes next, one branch to the other
if n.extra == next_label:
try self.branch_start("cbz ", "cbnz ", c)
try self.label_name(n.extra2)
try self.branch_end()
return
try self.branch_start("cbnz ", "cbz ", c)
try self.label_name(n.extra)
try self.branch_end()
if n.extra2 != next_label:
try self.writer.line("b ")
try self.label_name(n.extra2)
.switch_:
try self.jump_table(n)
.ret:
if n.extra2 > 0:
try self.return_words(n)
try self.return_value(n)
.copy:
if n.cls == ir.Class.v:
let v = try self.source(n.a, ir.Class.v, "q16")
let d = self.target(n.dst, ir.Class.v, "q16")
if v != d:
try self.writer.line4("mov ", vector_view(d), ".16b, ", vector_view(v))
try self.writer.put(".16b")
try self.finish(n.dst, ir.Class.v, d)
return
if (n.cls == ir.Class.w or n.cls == ir.Class.l) and n.a.kind == ir.RefKind.constant:
# a constant straight into where the temporary lives
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.load_immediate(d, n.a.value if n.cls == ir.Class.l else (i64)((u64)n.a.value & 0xFFFFFFFF))
try self.finish(n.dst, n.cls, d)
return
if n.a.kind == ir.RefKind.temp:
if let bits = self.fr.constant_of(n.a.index):
# a float constant formed where it is used: straight into the copy
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.float_constant(d, n.cls, bits)
try self.finish(n.dst, n.cls, d)
return
let v = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
if v != d:
if ir.is_float_class(n.cls):
try self.fmove(d, v)
else:
try self.writer.line3("mov ", d, ", ")
try self.writer.put(v)
try self.finish(n.dst, n.cls, d)
.add, .sub, .mul, .sdiv, .udiv, .srem, .urem, .and_, .or_, .xor, .shl, .lshr, .ashr:
if self.role() & (flags.joined | flags.folded) == 0:
try self.arithmetic(n)
.neg:
let v = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line("fneg " if ir.is_float_class(n.cls) else "neg ")
try self.writer.put(d)
try self.writer.put(", ")
try self.writer.put(v)
try self.finish(n.dst, n.cls, d)
.fsqrt, .ffloor, .fceil, .ftrunc, .fround, .fabs:
# the C library's float functions, each one instruction (`isa.has_float`)
let v = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line4(float_function_mnemonic(n.op), d, ", ", v)
try self.finish(n.dst, n.cls, d)
.ceq, .cne, .cslt, .csle, .csgt, .csge, .cult, .cule, .cugt, .cuge:
if self.role() & (flags.first | flags.either | flags.both) != 0:
try self.chained(n, self.role())
else:
try self.comparison(n)
.loadsb, .loadub, .loadsh, .loaduh, .loadsw, .loaduw, .loadl, .loads, .loadd, .loadhf:
let lc = ir.load_class(n.op)
let d = self.target(n.dst, lc, load_register(n.op))
let from_slot = self.fr.promoted_slot_of(n.a)
if from_slot != 0:
try self.load_promoted(n.op, from_slot, d)
elif let below = self.frame_place(n.a, n.op):
try self.frame_access(load_mnemonic(n.op), d, below, frame.access_width(n.op))
else:
let address = try self.source(n.a, ir.Class.l, "x9")
try self.writer.line(load_mnemonic(n.op))
try self.writer.put(" ")
try self.writer.put(d)
try self.writer.put(", [")
try self.writer.put(address)
try self.writer.put("]")
try self.finish(n.dst, lc, d)
.storeb, .storeh, .storew, .storel, .stores, .stored, .storehf:
let cls = ir.store_class(n.op)
let v = try self.source(n.a, cls, scratch_register(cls, 0))
let to_slot = self.fr.promoted_slot_of(n.b)
if to_slot != 0:
try self.store_promoted(n.op, to_slot, v)
elif let below = self.frame_place(n.b, n.op):
try self.frame_access(store_mnemonic(n.op), v, below, frame.access_width(n.op))
else:
let address = try self.source(n.b, ir.Class.l, "x10")
try self.writer.line(store_mnemonic(n.op))
try self.writer.put(" ")
try self.writer.put(v)
try self.writer.put(", [")
try self.writer.put(address)
try self.writer.put("]")
.alloca:
# frame storage (`memory.frame`): sp down by a multiple of sixteen, at most
# 4096, and the new bottom touched, so no guard page is stepped over
let bytes = try self.source(n.a, ir.Class.l, "x16")
try self.writer.line3("sub sp, sp, ", bytes, "")
try self.writer.line("str xzr, [sp]")
let d = self.target(n.dst, ir.Class.l, "x9")
try self.writer.line3("mov ", d, ", sp")
try self.finish(n.dst, ir.Class.l, d)
.frame:
let d = self.target(n.dst, ir.Class.l, "x9")
try self.writer.line("mov ")
try self.writer.put(d)
try self.writer.put(", x29")
try self.finish(n.dst, ir.Class.l, d)
.extsb, .extub, .extsh, .extuh, .extsw, .extuw:
let v = try self.source(n.a, ir.Class.w, "w9")
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
# a word already zero above, in the result's own register, is extended
let extended = n.op == ir.Op.extuw and n.a.kind == ir.RefKind.temp and (usize)n.a.index < self.zero_upper.length and self.zero_upper[(usize)n.a.index] and v == narrow_name(d)
if not extended:
try self.extend_from(n.op, v, d)
try self.finish(n.dst, n.cls, d)
.exth, .exts, .truncs, .truncd:
let src_cls = ir.float_source_class(n.op)
let v = try self.source(n.a, src_cls, scratch_register(src_cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line3("fcvt ", d, ", ")
try self.writer.put(v)
try self.finish(n.dst, n.cls, d)
.stosi, .stoui, .dtosi, .dtoui:
let src_cls = ir.Class.s if n.op == ir.Op.stosi or n.op == ir.Op.stoui else ir.Class.d
let v = try self.source(n.a, src_cls, scratch_register(src_cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line3("fcvtzs " if n.op == ir.Op.stosi or n.op == ir.Op.dtosi else "fcvtzu ", d, ", ")
try self.writer.put(v)
try self.finish(n.dst, n.cls, d)
.swtof, .uwtof, .sltof, .ultof:
let src_cls = ir.Class.w if n.op == ir.Op.swtof or n.op == ir.Op.uwtof else ir.Class.l
let v = try self.source(n.a, src_cls, scratch_register(src_cls, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
let convert = "scvtf " if n.op == ir.Op.swtof or n.op == ir.Op.sltof else "ucvtf "
try self.writer.line3(convert, d, ", ")
try self.writer.put(v)
try self.finish(n.dst, n.cls, d)
.cast:
if n.a.kind == ir.RefKind.constant and (n.cls == ir.Class.d or n.cls == ir.Class.s):
# a float literal straight into where the temporary lives
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.float_constant(d, n.cls, (u64)n.a.value)
try self.finish(n.dst, n.cls, d)
return
# bits move between an integer and a float of the same width
let other = ir.cast_partner(n.cls)
let v = try self.source(n.a, other, scratch_register(other, 0))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.fmove(d, v)
try self.finish(n.dst, n.cls, d)
.addr:
if self.fr.promoted_slot(n.dst) == 0:
let d = self.target(n.dst, ir.Class.l, "x9")
try self.address_operand(d, n.a)
try self.finish(n.dst, ir.Class.l, d)
.addr_tls:
try object.thread_address(&self.object_format, &self.writer.out, n.a.name)
try self.store_temp("x0", n.dst)
.blit:
if n.extra <= regalloc.inline_block:
# a record copy: loads and stores in place of a call, from and to the
# addresses where they live
let origin = try self.source(n.b, ir.Class.l, "x10")
let to = try self.source(n.a, ir.Class.l, "x11")
try self.inline_copy(origin, to, (u64)n.extra)
else:
# the origin first, through a scratch no value lives in: the destination's
# address may live in x0, and the origin's in x1
try self.address_operand("x10", n.b)
try self.address_operand("x0", n.a)
try self.writer.line("mov x1, x10")
try self.load_immediate("x2", (i64)n.extra)
try self.writer.line("bl ")
try self.symbol("memcpy")
.zero:
if n.extra <= regalloc.inline_block:
let to = try self.source(n.a, ir.Class.l, "x11")
try self.inline_zero(to, (u64)n.extra)
else:
try self.address_operand("x0", n.a)
try self.writer.line("mov x1, #0")
try self.load_immediate("x2", (i64)n.extra)
try self.writer.line("bl ")
try self.symbol("memset")
.fill:
try self.address_operand("x0", n.a)
try self.load_immediate("x1", n.b.value)
try self.load_immediate("x2", (i64)n.extra)
try self.writer.line("bl ")
try self.symbol("memset")
.call:
try self.call(n)
.word:
# moved out by the call before it (`take_words`)
_ = 0
.reserve:
try self.reserve_levels(n.extra)
.trap_:
if n.b.kind == ir.RefKind.text:
# the position first, then the reason (§11.5)
try self.address_operand("x0", n.b)
try self.address_operand("x1", n.a)
try self.writer.line("bl ")
try self.symbol(self.program_unit.trap_at_function)
else:
try self.address_operand("x0", n.a)
try self.call_trap()
.bounds:
let a = try self.source(n.a, ir.Class.l, "x9")
if n.b.kind == ir.RefKind.constant and n.b.value >= 0 and n.b.value <= 4095:
# a fixed array's length is the comparison's immediate
try self.writer.line3("cmp ", a, ", #")
try self.writer.put_i64(n.b.value)
else:
let b = try self.source(n.b, ir.Class.l, "x10")
try self.writer.line3("cmp ", a, ", ")
try self.writer.put(b)
try self.check_failure("b.ls " if n.extra == 1 else "b.lo ", "b.hi " if n.extra == 1 else "b.hs ", "index out of bounds", n)
.addo, .subo, .mulo:
try self.checked(n)
.asm_:
try self.inline_asm(n)
.set_reg:
# move an input operand into the exact register the asm block names
let reg = self.program_unit.texts.get((usize)n.b.index).bytes
try self.operand(reg, n.cls, n.a)
.get_reg:
# store the register the asm block left back into the output place; a
# discarded output has none
if n.a.kind == ir.RefKind.none_:
return
let reg = self.program_unit.texts.get((usize)n.b.index).bytes
try self.address_operand("x16", n.a)
try self.writer.line(store_mnemonic_for(n.cls))
try self.writer.put(" ")
try self.writer.put(reg)
try self.writer.put(", [x16]")
.atomic_load:
let address = try self.source(n.a, ir.Class.l, "x9")
let d = self.target(n.dst, ir.Class.w if n.extra <= 4 else ir.Class.l, scratch_register(ir.Class.w if n.extra <= 4 else ir.Class.l, 0))
try self.writer.line("ldar" if n.extra2 >= 2 else "ldr")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" ")
try self.writer.put(d)
try self.writer.put(", [")
try self.writer.put(address)
try self.writer.put("]")
try self.finish(n.dst, n.cls, d)
.atomic_store:
let cls = ir.Class.w if n.extra <= 4 else ir.Class.l
let v = try self.source(n.a, cls, scratch_register(cls, 0))
let address = try self.source(n.b, ir.Class.l, "x10")
try self.writer.line("stlr" if n.extra2 >= 3 else "str")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" ")
try self.writer.put(v)
try self.writer.put(", [")
try self.writer.put(address)
try self.writer.put("]")
.atomic_add, .atomic_sub, .atomic_or, .atomic_and_not, .atomic_xor, .atomic_swap, .atomic_max, .atomic_min:
try self.read_modify_write(n)
.atomic_cas:
try self.compare_and_swap(n)
.fence:
# a signal fence (6) orders the compiler alone: no instruction
if n.a.value != 6:
try self.writer.line("dmb ish")
.loadv, .storev, .vadd, .vsub, .vmul, .vdiv, .vand, .vor, .vxor, .vneg, .vnot, .vaddw, .vsubw, .vmulw, .vaddo, .vsubo, .vadds, .vsubs, .vshl, .vshr, .vsplat:
try self.vector(n)
.vinsert, .vextract, .vsqrt, .vexts, .vtruncd:
try self.packed_lanes(n)
.fma:
try self.fused(n)
.select:
try self.selected(n)
_:
error(failed, "an instruction the arm64 generator does not know")
func arithmetic(n: ir.Instr) -> !:
let a = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
if (n.cls == ir.Class.w or n.cls == ir.Class.l) and n.b.kind == ir.RefKind.constant:
let value = n.b.value
let width: u32 = 64 if n.cls == ir.Class.l else 32
let logical = (n.op == ir.Op.and_ or n.op == ir.Op.or_ or n.op == ir.Op.xor) and bitmask_immediate((u64)value, width)
# adding -c is subtracting c, and subtracting -c adding it
let negated = (n.op == ir.Op.add or n.op == ir.Op.sub) and value < 0 and value >= -4095
let immediate = ((n.op == ir.Op.add or n.op == ir.Op.sub) and value >= 0 and value <= 4095) or negated or ((n.op == ir.Op.shl or n.op == ir.Op.lshr or n.op == ir.Op.ashr) and value >= 0 and value < (64 if n.cls == ir.Class.l else 32)) or logical
let shift = power_of_two(value) if n.op == ir.Op.mul else 0
if n.op == ir.Op.mul and value > 2 and power_of_two(value - 1) > 0 and power_of_two(value - 1) < (i64)width:
# times 2^k + 1: x plus x shifted by k, one instruction a multiply's latency
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line5("add ", d, ", ", a, ", ")
try self.writer.put(a)
try self.writer.put(", lsl #")
try self.writer.put_i64(power_of_two(value - 1))
try self.finish(n.dst, n.cls, d)
return
if immediate or (n.op == ir.Op.mul and shift > 0 and shift < (i64)width):
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
var mnemonic = "add "
match n.op:
.add: mnemonic = "sub " if negated else "add "
.sub: mnemonic = "add " if negated else "sub "
.shl, .mul: mnemonic = "lsl "
.lshr: mnemonic = "lsr "
.ashr: mnemonic = "asr "
.and_: mnemonic = "and "
.or_: mnemonic = "orr "
.xor: mnemonic = "eor "
_: _ = 0
try self.writer.line4(mnemonic, d, ", ", a)
try self.writer.put(", #")
if logical:
# the mask as the assembler reads it: the operand's width of it, unsigned
try self.writer.put_u64((u64)value if width == 64 else (u64)value & 0xFFFFFFFF)
else:
try self.writer.put_i64(shift if n.op == ir.Op.mul else (0 - value if negated else value))
try self.finish(n.dst, n.cls, d)
return
let b = try self.source(n.b, n.cls, scratch_register(n.cls, 1))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
let is_float = ir.is_float_class(n.cls)
var mnemonic = "add"
match n.op:
.add: mnemonic = "fadd" if is_float else "add"
.sub: mnemonic = "fsub" if is_float else "sub"
.mul: mnemonic = "fmul" if is_float else "mul"
.sdiv: mnemonic = "fdiv" if is_float else "sdiv"
.udiv: mnemonic = "udiv"
.and_: mnemonic = "and"
.or_: mnemonic = "orr"
.xor: mnemonic = "eor"
.shl: mnemonic = "lsl"
.lshr: mnemonic = "lsr"
.ashr: mnemonic = "asr"
.srem, .urem:
try self.writer.line("sdiv " if n.op == ir.Op.srem else "udiv ")
try self.writer.put(scratch_register(n.cls, 2))
try self.writer.put(", ")
try self.writer.put(a)
try self.writer.put(", ")
try self.writer.put(b)
try self.writer.line("msub ")
try self.writer.put(d)
try self.writer.put(", ")
try self.writer.put(scratch_register(n.cls, 2))
try self.writer.put(", ")
try self.writer.put(b)
try self.writer.put(", ")
try self.writer.put(a)
try self.finish(n.dst, n.cls, d)
return
_: mnemonic = "add"
try self.writer.line(mnemonic)
try self.writer.put(" ")
try self.writer.put(d)
try self.writer.put(", ")
try self.writer.put(a)
try self.writer.put(", ")
try self.writer.put(b)
try self.finish(n.dst, n.cls, d)
func comparison(n: ir.Instr, branch: ir.Instr? = none, next_label: u32 = 0, negate: bool = false) -> !:
let a = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let is_float = ir.is_float_class(n.cls)
if let jump = branch:
if not is_float and (n.op == ir.Op.ceq or n.op == ir.Op.cne) and n.b.kind == ir.RefKind.constant and n.b.value == 0:
# a branch on a value against zero tests the register itself
let zero = n.op == ir.Op.ceq
try self.branch_start("cbz " if zero else "cbnz ", "cbnz " if zero else "cbz ", a)
try self.label_name(jump.extra2 if next_label == jump.extra else jump.extra)
try self.branch_end()
if next_label != jump.extra and next_label != jump.extra2:
try self.writer.line("b ")
try self.label_name(jump.extra2)
return
# an integer constant the instruction can carry: `cmp` up to 4095, `cmn` below 0
let immediate = not is_float and n.b.kind == ir.RefKind.constant and n.b.value >= -4095 and n.b.value <= 4095
let b = "" if immediate else try self.source(n.b, n.cls, scratch_register(n.cls, 1))
if immediate:
try self.writer.line3("cmp " if n.b.value >= 0 else "cmn ", a, ", #")
try self.writer.put_i64(n.b.value if n.b.value >= 0 else 0 - n.b.value)
else:
try self.writer.line("fcmp " if is_float else "cmp ")
try self.writer.put(a)
try self.writer.put(", ")
try self.writer.put(b)
var cond = condition(n.op, is_float)
if negate:
cond = opposite_condition(cond)
if let jump = branch:
try self.branch_on(cond)
try self.label_name(jump.extra2 if next_label == jump.extra else jump.extra)
try self.branch_end()
if next_label != jump.extra and next_label != jump.extra2:
try self.writer.line("b ")
try self.label_name(jump.extra2)
return
let d = self.target(n.dst, ir.Class.w, "w9")
try self.writer.line3("cset ", d, ", ")
try self.writer.put(cond)
try self.finish(n.dst, ir.Class.w, d)
## `switch`: the value less the table's first is the entry, when it is below the
## table's count; the entry is its label's distance from the table, which follows the
## branch in the text, so the table needs no relocation. x9 holds the entry, x10 the
## table's address.
func jump_table(n: ir.Instr) -> !:
let wide = n.cls == ir.Class.l
let table = self.f.jump_tables.get((usize)n.extra2)
let v = try self.source(n.a, n.cls, "x9" if wide else "w9")
let scratch = "x9" if wide else "w9"
# the low bits are the whole value at the word's width
let low = (u64)table.low if wide else (u64)table.low & 0xFFFFFFFF
let modulus = 0 -% low if wide else ((u64)1 << 32) - low
var index = v
if low == 0:
_ = 0
elif low <= 4095:
try self.writer.line5("sub ", scratch, ", ", v, ", #")
try self.writer.put_u64(low)
index = scratch
elif modulus <= 4095:
try self.writer.line5("add ", scratch, ", ", v, ", #")
try self.writer.put_u64(modulus)
index = scratch
else:
try self.load_immediate("x10", (i64)low)
try self.writer.line5("sub ", scratch, ", ", v, ", ")
try self.writer.put("x10" if wide else "w10")
index = scratch
try self.writer.line3("cmp ", index, ", #")
try self.writer.put_u64((u64)table.count)
try self.branch_on("hs")
try self.label_name(n.extra)
try self.branch_end()
try self.writer.line("adr x10, ")
try self.table_name(n.extra2)
try self.writer.line3("ldrsw x9, [x10, ", index, ", lsl #2]" if wide else ", uxtw #2]")
try self.writer.line("add x10, x10, x9")
try self.writer.line("br x10")
try self.writer.put("\n")
try self.table_name(n.extra2)
try self.writer.put(":")
for label in self.f.table_targets(n.extra2):
try self.writer.line(".long ")
try self.label_name(label)
try self.writer.put(" - ")
try self.table_name(n.extra2)
## The label of jump table `n` of the function.
func table_name(n: u32) -> !:
try self.writer.out.put(self.object_format.local_label)
try stable_names.put_function_label(&self.writer.out, self.index, self.function_tags, self.split or self.stable)
try self.writer.out.put("_jt")
try self.writer.out.put_u64((u64)n)
## `op` applied to the 32-bit view `v` of a value, into `d`.
func extend_from(op: ir.Op, v: str, d: str) -> !:
match op:
.extsb: try self.writer.line4("sxtb ", d, ", ", v)
.extub: try self.writer.line5("and ", d, ", ", v, ", #255")
.extsh: try self.writer.line4("sxth ", d, ", ", v)
.extuh: try self.writer.line5("and ", d, ", ", v, ", #65535")
.extsw: try self.writer.line4("sxtw ", d, ", ", v)
_:
# zero-extension of a word: writing the 32-bit view clears the upper half,
# also when the source is that same register, whose upper half may hold a
# sign extension or what a wider write left there
try self.writer.line4("mov ", narrow_name(d), ", ", v)
## Checked add, subtract, and multiply: trap on overflow of the class width.
## `select`: the condition word against 0, then one `csel` (`fcsel` for a float) of the
## two values; `extra` is 1 when the condition is a whole word, 0 for its low half. The
## flags already hold the condition after its chain (`flags`); a choice between `x` and
## `x + 1` is `cinc`, whose `add` was not selected.
func selected(n: ir.Instr) -> !:
let role = self.role()
let other = self.f.args.get((usize)n.extra2).value
var cond = "ne"
if role & flags.reads != 0:
cond = self.flag_condition
else:
let wide = ir.Class.l if n.extra == 1 else ir.Class.w
let c = try self.source(n.a, wide, "x9" if n.extra == 1 else "w9")
try self.writer.line3("cmp ", c, ", #0")
if role & (flags.then_increments | flags.else_increments) != 0:
# the value that is not the sum, and whether the sum is taken on the condition
let then_sum = role & flags.then_increments != 0
let v = try self.source(other if then_sum else n.b, n.cls, scratch_register(n.cls, 2))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line5("cinc ", d, ", ", v, ", ")
try self.writer.put(cond if then_sum else opposite_condition(cond))
try self.finish(n.dst, n.cls, d)
return
let t = try self.source(n.b, n.cls, scratch_register(n.cls, 1))
let e = try self.source(other, n.cls, scratch_register(n.cls, 2))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line4("csel " if n.cls == ir.Class.w or n.cls == ir.Class.l else "fcsel ", d, ", ", t)
try self.writer.put(", ")
try self.writer.put(e)
try self.writer.put(", ")
try self.writer.put(cond)
try self.finish(n.dst, n.cls, d)
## The role in a condition kept in the flags of the instruction being emitted.
func role() -> u8:
return self.flag_roles[self.at] if self.at < self.flag_roles.length else flags.plain
## A comparison of a chain kept in the flags (`flags`): the first `cmp` (`fcmp`), each
## later one `ccmp` (`fccmp`), which compares only while the chain so far leaves the
## answer open (`or`: while it is false; `and`: while it is true) and otherwise sets
## flags that settle it.
func chained(n: ir.Instr, role: u8) -> !:
let is_float = ir.is_float_class(n.cls)
let a = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let opening = role & flags.first != 0
# `cmp` carries 0 to 4095 (`cmn` their negations), `ccmp` 0 to 31, a float none
let low: i64 = -4095 if opening else 0
let high: i64 = 4095 if opening else 31
let immediate = not is_float and n.b.kind == ir.RefKind.constant and n.b.value >= low and n.b.value <= high
let b = "" if immediate else try self.source(n.b, n.cls, scratch_register(n.cls, 1))
let cond = condition(n.op, is_float)
if opening:
let mnemonic = "fcmp " if is_float else ("cmn " if immediate and n.b.value < 0 else "cmp ")
try self.writer.line3(mnemonic, a, ", ")
else:
try self.writer.line3("fccmp " if is_float else "ccmp ", a, ", ")
if immediate:
try self.writer.put("#")
try self.writer.put_i64(n.b.value if n.b.value >= 0 else 0 - n.b.value)
else:
try self.writer.put(b)
if not opening:
let either = role & flags.either != 0
try self.writer.put(", #")
try self.writer.put_u64(settling_flags(cond, either))
try self.writer.put(", ")
try self.writer.put(opposite_condition(self.flag_condition) if either else self.flag_condition)
self.flag_condition = cond
## `fma`: `fmadd` for a scalar; for a vector, `fmla`, which accumulates into its
## destination: the result's register takes the addend first, or v18 when that register
## holds a factor.
func fused(n: ir.Instr) -> !:
let addend = self.f.args.get((usize)n.extra2).value
if n.cls != ir.Class.v:
let a = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
let b = try self.source(n.b, n.cls, scratch_register(n.cls, 1))
let c = try self.source(addend, n.cls, scratch_register(n.cls, 2))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line4("fmadd ", d, ", ", a)
try self.writer.put(", ")
try self.writer.put(b)
try self.writer.put(", ")
try self.writer.put(c)
try self.finish(n.dst, n.cls, d)
return
let arr = arrangement(n.extra)
let a = vector_view(try self.source(n.a, ir.Class.v, "q16"))
let b = vector_view(try self.source(n.b, ir.Class.v, "q17"))
let c = vector_view(try self.source(addend, ir.Class.v, "q18"))
# the result's own register takes the addend, unless it holds a factor still to read
var d = self.target(n.dst, ir.Class.v, "q18")
if vector_view(d) == a or vector_view(d) == b:
d = "q18"
let accumulator = vector_view(d)
if c != accumulator:
try self.writer.line5("mov ", accumulator, ".16b, ", c, ".16b")
try self.writer.line5("fmla ", accumulator, arr, ", ", a)
try self.writer.put(arr)
try self.writer.put(", ")
try self.writer.put(b)
try self.writer.put(arr)
if d == "q18" and self.target(n.dst, ir.Class.v, "q18") != "q18":
d = self.target(n.dst, ir.Class.v, "q18")
try self.writer.line3("mov ", vector_view(d), ".16b, v18.16b")
try self.finish(n.dst, ir.Class.v, d)
## A checked operation reads its operands where they live and computes into the
## destination's register when it has one; a multiply computes in x9 (its high word
## or widened check in x10 and x11) and moves the product out.
func checked(n: ir.Instr) -> !:
let signed = n.extra == 1
let a = try self.source(n.a, n.cls, scratch_register(n.cls, 0))
# `adds x1, x2, #8`: an addend or subtrahend of 0..4095 is the instruction's own
let immediate = n.op != ir.Op.mulo and n.b.kind == ir.RefKind.constant and n.b.value >= 0 and n.b.value <= 4095
let b = "" if immediate else try self.source(n.b, n.cls, scratch_register(n.cls, 1))
if n.op == ir.Op.mulo:
if n.cls == ir.Class.w:
# multiply at 64 bits and check the range
if signed:
try self.writer.line3("sxtw x9, ", a, "")
try self.writer.line3("sxtw x10, ", b, "")
try self.writer.line("mul x9, x9, x10")
try self.writer.line("cmp x9, w9, sxtw")
else:
try self.writer.line3("mov w9, ", a, "")
try self.writer.line3("mov w10, ", b, "")
try self.writer.line("mul x9, x9, x10")
try self.writer.line("cmp x9, w9, uxtw")
try self.check_failure("b.eq ", "b.ne ", "integer overflow", n)
else:
# the high half first, then the product straight into the result's register,
# which may be an operand's: both have been read by then
let d = self.target(n.dst, n.cls, "x9")
try self.writer.line3("smulh x11, " if signed else "umulh x11, ", a, ", ")
try self.writer.put(b)
try self.product(d, a, b, n.b)
if signed:
try self.writer.line3("cmp x11, ", d, ", asr #63")
try self.check_failure("b.eq ", "b.ne ", "integer overflow", n)
else:
try self.check_failure("cbz x11, ", "cbnz x11, ", "integer overflow", n)
try self.finish(n.dst, n.cls, d)
return
try self.finish(n.dst, n.cls, scratch_register(n.cls, 0))
return
let add = n.op == ir.Op.addo
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line("adds " if add else "subs ")
try self.writer.put(d)
try self.writer.put(", ")
try self.writer.put(a)
try self.writer.put(", ")
if immediate:
try self.writer.put("#")
try self.writer.put_i64(n.b.value)
else:
try self.writer.put(b)
if signed:
try self.check_failure("b.vc ", "b.vs ", "integer overflow", n)
elif add:
try self.check_failure("b.cc ", "b.cs ", "integer overflow", n)
else:
try self.check_failure("b.cs ", "b.cc ", "integer overflow", n)
try self.finish(n.dst, n.cls, d)
## `d = a * b`, the low half of a checked product whose high half decided the check: by
## a constant 3, 5 or 9 an add of `a` shifted, by a power of two a shift, neither of
## which waits on the multiplier; `factor` is the right operand as the IR names it.
func product(d: str, a: str, b: str, factor: ir.Ref) -> !:
let c = factor.value if factor.kind == ir.RefKind.constant else 0
if c == 3 or c == 5 or c == 9:
try self.writer.line5("add ", d, ", ", a, ", ")
try self.writer.put(a)
try self.writer.put(", lsl #")
try self.writer.put_u64(1 if c == 3 else (2 if c == 5 else 3))
return
if c > 1 and (c & (c - 1)) == 0:
var shift: u64 = 0
while ((i64)1 << (i64)shift) != c:
shift += 1
try self.writer.line5("lsl ", d, ", ", a, ", #")
try self.writer.put_u64(shift)
return
try self.writer.line5("mul ", d, ", ", a, ", ")
try self.writer.put(b)
## Where a check of `n` that fails goes, after the comparison that decides it: with
## `--debug` to its trap inline, a branch on `pass` jumping over it; otherwise a branch
## on `fail` to a path after the function's body (`finish_cold`), so the passing path
## falls through and a loop's body holds no trap's instructions. `pass` and `fail` are
## the branch before its label (`b.lo `, `cbz x11, `).
func check_failure(pass: str, fail: str, reason: str, n: ir.Instr) -> !:
if self.f.debug:
try self.writer.line2(pass, "1f")
try self.trap_text(reason, n.extra2)
try self.writer.put("\n1:")
return
self.cold_count += 1
try self.branch_start(fail, pass)
try self.cold_label(self.cold_count)
try self.branch_end()
# the trap goes to the cold paths' text, through the writer every emission uses
let body = self.writer.out
self.writer.out = self.cold
try self.writer.put("\n")
try self.cold_label(self.cold_count)
try self.writer.put(":")
if let dw = self.dw:
try dw.line_of(&self.writer.out, self.f, n)
try self.trap_text(reason, n.extra2)
self.cold = self.writer.out
self.writer.out = body
## The start of a conditional branch, up to its label: `taken`, then `operand` and a
## comma when there is one (`cbz ` and `x9`; `b.hs `). In a function a conditional branch
## may not reach across (`short_reach`), `skipped`, the opposite branch, jumps over an
## unconditional `b` to the label instead; `branch_end` places its target.
func branch_start(taken: str, skipped: str, operand: str = "") -> !:
let far = self.f.instrs.count > short_reach
try self.writer.line(skipped if far else taken)
if operand.length > 0:
try self.writer.put(operand)
try self.writer.put(", ")
if far:
try self.writer.put("1f")
try self.writer.line("b ")
## `branch_start` for `b.cond`, on condition `cond` (`hs`).
func branch_on(cond: str) -> !:
let far = self.f.instrs.count > short_reach
try self.writer.line("b.")
try self.writer.put(opposite_condition(cond) if far else cond)
try self.writer.put(" 1f" if far else " ")
if far:
try self.writer.line("b ")
## The end of a conditional branch after its label: the target of a far one's skip.
func branch_end() -> !:
if self.f.instrs.count > short_reach:
try self.writer.put("\n1:")
## The function's cold paths (`check_failure`), after its body and before its end.
func finish_cold() -> !:
if self.cold.used > 0:
try self.writer.put(self.cold.view())
self.cold.clear()
self.cold_count = 0
## A trap with a fixed reason; `at`, when not zero, is the unit's text of the statement's
## position plus one, which the reporter names first (§11.5).
func trap_text(reason: str, at: u32) -> !:
# the reasons were added before the functions were generated (`trap_reasons`), so
# the threads generating them only read the unit's texts
let index = self.program_unit.find_text(reason) else error(ir.missing_text, "a trap's reason is not among the unit's texts")
if at > 0:
try self.address_operand("x0", ir.text_ref(at - 1))
try self.address_operand("x1", ir.text_ref(index))
try self.writer.line("bl ")
try self.symbol(self.program_unit.trap_at_function)
return
try self.address_operand("x0", ir.text_ref(index))
try self.call_trap()
## The program's trap reporter, with the reason in x0.
func call_trap() -> !:
try self.writer.line("bl ")
try self.symbol(self.program_unit.trap_function)
## The instruction of a C library float function (`isa.has_float`): `frintm` rounds toward
## negative infinity, `frintp` toward positive, `frintz` toward zero, and `frinta` to nearest
## with halves away from zero, as `floor`, `ceil`, `trunc` and `round` do.
func float_function_mnemonic(op: ir.Op) -> str:
match op:
.fsqrt: return "fsqrt "
.ffloor: return "frintm "
.fceil: return "frintp "
.ftrunc: return "frintz "
.fround: return "frinta "
_: return "fabs "