dymokomi/luce-base
Source of release 0.44.0
0.44.0 / src / back / native / arm64 / registers.lucb
#==============================================================================================
#
# arm64/registers - Registers and immediates
#
# DESCRIPTION:
# The register each class and number names, immediates that fit, and a slot's
# or a temporary's place in the frame.
#
#==============================================================================================
extend Generator:
# mark: Registers and immediates ===========================================================
## `movz`/`movk` for any 64-bit value.
func load_immediate(reg: str, value: i64) -> !:
let bits = (u64)value
if bits == 0:
try self.writer.line("mov ")
try self.writer.put(reg)
try self.writer.put(", #0")
return
# a negative number is shorter with movn
if value < 0 and value >= -65536:
try self.writer.line("movn ")
try self.writer.put(reg)
try self.writer.put(", #")
try self.writer.put_u64((u64)(-1 - value))
return
var first = true
var shift: u64 = 0
while shift < 64:
let piece = (bits >> shift) & 65535
if piece != 0 or (first and shift == 48):
try self.writer.line("movz " if first else "movk ")
try self.writer.put(reg)
try self.writer.put(", #")
try self.writer.put_u64(piece)
if shift > 0:
try self.writer.put(", lsl #")
try self.writer.put_u64(shift)
first = false
shift += 16
## The memory operand `[sp, #n]` for the frame home `offset` bytes below x29, for an
## access of `width` bytes: the scaled unsigned form reaches 4095 units of the width.
## The frame is laid out before any instruction is emitted, so `frame` is known.
func frame_fits(offset: u64, width: u64) -> bool:
if offset > self.fr.size:
return false
let n = self.fr.size - offset + self.frame_bias()
return n % width == 0 and n // width <= 4095
func frame_operand(offset: u64) -> !:
try self.writer.put("[x28, #" if self.frame_base else "[sp, #")
try self.writer.put_u64(self.fr.size - offset + self.frame_bias())
try self.writer.put("]")
## How far below the frame's bottom sp is: none through x28 (`frame_base`).
func frame_bias() -> u64:
if self.frame_base:
return 0
return self.sp_shift + self.dynamic_shift
## `reg` stored to or loaded from the frame home `offset`: through sp when the offset
## reaches, else through an address in x16.
## A half register stores through its single view, so its home's whole word is written
## and a reinterpretation as an integer reads zeros above the half.
func frame_store(reg: str, offset: u64) -> !:
let spelled = single_view(reg)
if self.frame_fits(offset, access_width_of(spelled)):
try self.writer.line3("str ", spelled, ", ")
try self.frame_operand(offset)
return
try self.frame_address("x16", offset)
try self.writer.line3("str ", spelled, ", [x16]")
func frame_load(reg: str, offset: u64) -> !:
if self.frame_fits(offset, access_width_of(reg)):
try self.writer.line3("ldr ", reg, ", ")
try self.frame_operand(offset)
return
try self.frame_address("x16", offset)
try self.writer.line3("ldr ", reg, ", [x16]")
## How far below the frame pointer the bytes that `address` names lie, when it is a
## slot or a frame address formed at its use (`remat`), for an access by `op` that
## addresses the frame directly; none for any other address, or a half's access.
func frame_place(address: ir.Ref, op: ir.Op) -> u64?:
if op == ir.Op.loadhf or op == ir.Op.storehf:
return none
if address.kind == ir.RefKind.slot:
return self.slot_home(address.index)
if address.kind == ir.RefKind.temp and self.fr.temp_regs.get((usize)address.index - 1) == 0:
return self.fr.rematerialised(address.index)
return none
## A load or store at `base + offset`, the add that formed the address folded into it
## (`selection.offset_access`).
func offset_access(n: ir.Instr, base: ir.Ref, offset: i64) -> !:
if (u8)n.op <= (u8)ir.Op.loadd:
let lc = ir.load_class(n.op)
let d = self.target(n.dst, lc, load_register(n.op))
let b = try self.source(base, ir.Class.l, "x10")
try self.writer.line4(load_mnemonic(n.op), " ", d, ", [")
try self.writer.put(b)
try self.writer.put(", #")
try self.writer.put_i64(offset)
try self.writer.put("]")
try self.finish(n.dst, lc, d)
return
let cls = ir.store_class(n.op)
let v = try self.source(n.a, cls, scratch_register(cls, 0))
let b = try self.source(base, ir.Class.l, "x10")
try self.writer.line4(store_mnemonic(n.op), " ", v, ", [")
try self.writer.put(b)
try self.writer.put(", #")
try self.writer.put_i64(offset)
try self.writer.put("]")
## Emit instructions `at` to `at + 2` as one scaled-index access when they are one
## (`selection.scaled_access`), and the widening after when the access is a signed
## word's load (`access_at`); how many instructions that was.
func scaled_access_at(at: usize, uses: u32[], index: usize) -> usize?!:
let scaled = selection.scaled_access(self.f, at, uses) else return none
let (base, scaled_index, shift) = scaled
if self.fr.is_rematerialised(self.f.instrs.get(at).dst) or self.fr.is_rematerialised(self.f.instrs.get(at + 1).dst):
return none
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, at + 2)
let (access, widened) = self.access_at(at + 2, uses)
try self.indexed_access(access, base, scaled_index, shift)
return 3 + widened
## `add d, other, x, lsl #shift` for a shift and the add of it after
## (`selection.shifted_add`).
func shifted_add(n: ir.Instr, fused: (ir.Ref, ir.Ref, u32)) -> !:
let (other, shifted, shift) = fused
let o = try self.source(other, n.cls, scratch_register(n.cls, 0))
let x = try self.source(shifted, n.cls, scratch_register(n.cls, 1))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line5("add ", d, ", ", o, ", ")
try self.writer.put(x)
try self.writer.put(", lsl #")
try self.writer.put_u64((u64)shift)
try self.finish(n.dst, n.cls, d)
## The add `n` of a product by a constant right before it (`selection.multiplied_add`),
## as one `madd`: the variable factor in the first scratch register when not in its own,
## the constant formed in the second, the other addend in the third.
func multiplied_add(n: ir.Instr, fused: (ir.Ref, ir.Ref, ir.Ref)) -> !:
let (other, a, b) = fused
let x = try self.source(a, n.cls, scratch_register(n.cls, 0))
let y = scratch_register(n.cls, 1)
try self.load_immediate(y, b.value if n.cls == ir.Class.l else (i64)((u64)b.value & 0xFFFFFFFF))
let o = try self.source(other, n.cls, scratch_register(n.cls, 2))
let d = self.target(n.dst, n.cls, scratch_register(n.cls, 0))
try self.writer.line5("madd ", d, ", ", x, ", ")
try self.writer.put(y)
try self.writer.put(", ")
try self.writer.put(o)
try self.finish(n.dst, n.cls, d)
## Emit instructions `at` and `at + 1` as one register-offset access when they are one
## (`selection.summed_access`), and the widening after as `scaled_access_at` does; how
## many instructions that was.
func summed_access_at(at: usize, uses: u32[], index: usize) -> usize?!:
let summed = selection.summed_access(self.f, at, uses) else return none
let (base, offset) = summed
if self.fr.is_rematerialised(self.f.instrs.get(at).dst):
return none
if let dw = self.dw:
try dw.mark(&self.writer.out, self.f, &self.fr, index, at + 1)
let (access, widened) = self.access_at(at + 1, uses)
try self.indexed_access(access, base, offset, 0)
return 2 + widened
## A load or store through `base + (index << shift)` (`selection.scaled_access`), the
## scaling and the sum done by the addressing: `ldr w0, [x1, x2, lsl #2]`.
func indexed_access(n: ir.Instr, base: ir.Ref, index: ir.Ref, shift: u32) -> !:
let b = try self.source(base, ir.Class.l, "x10")
let x = try self.source(index, ir.Class.l, "x11")
if (u8)n.op <= (u8)ir.Op.loadd:
let lc = ir.load_class(n.op)
let d = self.target(n.dst, lc, load_register(n.op))
try self.writer.line4(load_mnemonic(n.op), " ", d, ", [")
try self.writer.put(b)
try self.writer.put(", ")
try self.writer.put(x)
if shift != 0:
try self.writer.put(", lsl #")
try self.writer.put_u64((u64)shift)
try self.writer.put("]")
try self.finish(n.dst, lc, d)
return
let cls = ir.store_class(n.op)
let v = try self.source(n.a, cls, scratch_register(cls, 0))
try self.writer.line4(store_mnemonic(n.op), " ", v, ", [")
try self.writer.put(b)
try self.writer.put(", ")
try self.writer.put(x)
if shift != 0:
try self.writer.put(", lsl #")
try self.writer.put_u64((u64)shift)
try self.writer.put("]")
## `mnemonic reg` to or from the frame `below` bytes under x29, `width` bytes wide:
## through sp when the scaled offset reaches, else x29 with an unscaled one, else an
## address in x16.
func frame_access(mnemonic: str, reg: str, below: u64, width: u32) -> !:
if self.frame_fits(below, (u64)width):
try self.writer.line4(mnemonic, " ", reg, ", ")
try self.frame_operand(below)
elif below <= 256:
try self.writer.line4(mnemonic, " ", reg, ", [x29, #-")
try self.writer.put_u64(below)
try self.writer.put("]")
else:
try self.frame_address("x16", below)
try self.writer.line4(mnemonic, " ", reg, ", [x16]")
## `reg = x29 - offset`.
func frame_address(reg: str, offset: u64) -> !:
if offset <= 4095:
try self.writer.line("sub ")
try self.writer.put(reg)
try self.writer.put(", x29, #")
try self.writer.put_u64(offset)
return
try self.load_immediate("x16", (i64)offset)
try self.writer.line("sub ")
try self.writer.put(reg)
try self.writer.put(", x29, x16")
## Before the barrier at instruction `at`: each temporary that keeps a caller-saved
## register across it goes to its home (`frame.Frame.crossings`).
func save_crossing(at: usize) -> !:
while self.crossing < self.fr.crossings.count and (usize)self.fr.crossings.get(self.crossing).at < at:
self.crossing += 1
var k = self.crossing
while k < self.fr.crossings.count and (usize)self.fr.crossings.get(k).at == at:
let c = self.fr.crossings.get(k)
let code = self.fr.temp_regs.get((usize)c.temp - 1)
let cls = self.f.temp_class(c.temp)
try self.frame_store(class_name(code, cls), self.temp_home(c.temp))
k += 1
## After it: each comes back from its home.
func restore_crossing(at: usize) -> !:
while self.crossing < self.fr.crossings.count and (usize)self.fr.crossings.get(self.crossing).at == at:
let c = self.fr.crossings.get(self.crossing)
let code = self.fr.temp_regs.get((usize)c.temp - 1)
let cls = self.f.temp_class(c.temp)
try self.frame_load(class_name(code, cls), self.temp_home(c.temp))
self.crossing += 1
func temp_home(t: u32) -> u64:
return self.fr.temp_home(t)
func slot_home(s: u32) -> u64:
return self.fr.slot_home(s)
## A homogeneous floating-point aggregate (AAPCS64 §6.4.2): one to four floats of one
## width, one after the other with nothing between; how many, or 0 when the shape is
## not one.
func hfa(shape: u32) -> u32:
let leaves = self.program_unit.leaves_of(shape)
if leaves.length == 0 or leaves.length > 4:
return 0
let first = leaves[0]
if not ir.is_float_class(first.cls):
return 0
var k: usize = 0
while k < leaves.length:
let leaf = leaves[k]
if leaf.cls != first.cls or leaf.size != first.size or (u64)leaf.offset != (u64)k * (u64)first.size:
return 0
k += 1
return u32(leaves.length)
## Load a temporary into `reg`: a register move when it lives in one, else from its home.
## A float register moves with `fmov`, an integer one with `mov`; the width follows `reg`.
## The register holding `r` as class `cls` when `r` is a temporary living in one of the
## right kind, else `scratch` after loading it there. Instructions then read operands
## where they are rather than through a scratch register.
func source(r: ir.Ref, cls: ir.Class, scratch: str) -> str!:
if r.kind == ir.RefKind.temp:
let code = self.fr.temp_regs.get((usize)r.index - 1)
if code != 0 and self.fr.float_code(code) == (ir.is_float_class(cls) or cls == ir.Class.v):
return class_name(code, cls)
try self.operand(scratch, cls, r)
return scratch
## The register to compute temporary `dst` into: its own when it lives in one of the
## right kind, else `scratch`, which `finish` then stores.
func target(dst: u32, cls: ir.Class, scratch: str) -> str:
if dst == 0:
return scratch
let code = self.fr.temp_regs.get((usize)dst - 1)
if code != 0 and self.fr.float_code(code) == (ir.is_float_class(cls) or cls == ir.Class.v):
return class_name(code, cls)
return scratch
## After a result landed in `reg`: nothing when that is the temporary's own register.
func finish(dst: u32, cls: ir.Class, reg: str) -> !:
if dst == 0:
return
let code = self.fr.temp_regs.get((usize)dst - 1)
if code != 0 and class_name(code, cls) == reg:
return
try self.store_temp(reg, dst)
func load_temp(reg: str, cls: ir.Class, t: u32) -> !:
let code = self.fr.temp_regs.get((usize)t - 1)
if code != 0:
let home = register_view(reg, code)
if home != reg:
try self.register_move(reg, home, code)
return
if let bits = self.fr.constant_of(t):
# a float constant, formed here (`frame.Frame.constant_of`)
try self.float_constant(reg, cls, bits)
return
if let below = self.fr.rematerialised(t):
# a frame address, formed here (`remat`)
if reg.bytes[0] == 'x' or reg.bytes[0] == 'w':
try self.frame_address(as_x(reg), below)
else:
try self.frame_address("x16", below)
try self.fmove(reg, "x16")
return
try self.frame_load(reg, self.temp_home(t))
func store_temp(reg: str, t: u32) -> !:
let code = self.fr.temp_regs.get((usize)t - 1)
if code != 0:
let home = register_view(reg, code)
if home != reg:
try self.register_move(home, reg, code)
return
try self.frame_store(reg, self.temp_home(t))
## `fmov to, from` between float registers, halves included (FEAT_FP16).
func fmove(target_reg: str, from_reg: str) -> !:
try self.writer.line("fmov ")
try self.writer.put(target_reg)
try self.writer.put(", ")
try self.writer.put(from_reg)
## `to = from` between registers: `mov` for integers, `fmov` for floats, and the whole
## vector for a `q` view.
func register_move(target_reg: str, from_reg: str, code: u8) -> !:
if target_reg.bytes[0] == 'q':
try self.writer.line("mov ")
try self.writer.put(vector_view(target_reg))
try self.writer.put(".16b, ")
try self.writer.put(vector_view(from_reg))
try self.writer.put(".16b")
return
if self.fr.float_code(code):
try self.fmove(target_reg, from_reg)
return
try self.writer.line("mov ")
try self.writer.put(target_reg)
try self.writer.put(", ")
try self.writer.put(from_reg)
## An operand into a scratch register of class `cls`.
func operand(reg: str, cls: ir.Class, r: ir.Ref) -> !:
if r.kind == ir.RefKind.temp:
let tc = self.f.temp_class(r.index)
if tc == cls or (is_int_class(tc) and is_int_class(cls)):
try self.load_temp(reg, cls, r.index)
return
# a float temporary read as an integer word or the reverse: reinterpret the home's bytes
try self.load_temp(reg, cls, r.index)
return
if r.kind == ir.RefKind.constant:
if ir.is_float_class(cls):
try self.float_constant(reg, cls, (u64)r.value)
return
try self.load_immediate(as_x(reg), r.value)
return
try self.address_operand(as_x(reg), r)
## The address a slot, global, function, or text refers to, into an x register.
func address_operand(reg: str, r: ir.Ref) -> !:
if r.kind == ir.RefKind.slot:
try self.frame_address(reg, self.slot_home(r.index))
elif r.kind == ir.RefKind.global and r.value == 0:
try object.page_address(&self.object_format, &self.writer.out, reg, r.name, true)
elif r.kind == ir.RefKind.global or r.kind == ir.RefKind.function:
# a function's address, or another image's data (`import_ref`), goes through the
# GOT, so a symbol defined in a linked shared library resolves
try object.got_address(&self.object_format, &self.writer.out, reg, r.name)
elif r.kind == ir.RefKind.text:
try object.page_address(&self.object_format, &self.writer.out, reg, try self.text_label_name(r.index), false)
elif r.kind == ir.RefKind.temp:
try self.load_temp(reg, ir.Class.l, r.index)
elif r.kind == ir.RefKind.constant:
try self.load_immediate(reg, r.value)
else:
try self.writer.line("mov ")
try self.writer.put(reg)
try self.writer.put(", #0")
# mark: Float constants ===================================================================
## The float constant of `bits` (a single's in the low half) into `reg` of class `cls`:
## positive zero from the zero register, a value `fmov` encodes as its immediate, any
## other loaded from the function's literal pool by one `ldr`, where the movz/movk
## sequence and `fmov` it replaces took up to five. An integer register (the bits read
## as an integer) and a half take the bits through an immediate. A function too large
## for a literal load to reach past its end builds every constant so.
func float_constant(reg: str, cls: ir.Class, bits: u64) -> !:
let first = reg.bytes[0]
if first == 'x' or first == 'w' or cls == ir.Class.h or (cls != ir.Class.d and cls != ir.Class.s):
if first == 'x' or first == 'w':
try self.load_immediate(as_x(reg), (i64)bits)
return
try self.load_immediate("x16", (i64)bits)
try self.fmove(reg, "x16" if cls == ir.Class.d else "w16")
return
let wide = cls == ir.Class.d
if bits == 0:
try self.writer.line4("fmov ", reg, ", ", "xzr" if wide else "wzr")
return
if let scaled = fmov_scaled(bits, wide):
try self.writer.line3("fmov ", reg, ", #")
try self.put_scaled(scaled)
return
if self.f.instrs.count > short_reach:
try self.load_immediate("x16", (i64)bits)
try self.fmove(reg, "x16" if wide else "w16")
return
var k: usize = 0
while k < self.pool.count and self.pool.get(k) != bits:
k += 1
if k == self.pool.count:
try self.pool.push(bits)
try self.writer.line3("ldr ", reg, ", ")
try self.pool_label(u32(k))
## A value times 128 (`fmov_scaled`) in decimal, exactly: its whole part, a point, and
## seven digits of its 128ths (`2.5000000`), as the assembler reads an `fmov` immediate.
func put_scaled(scaled: i64) -> !:
let magnitude = (u64)(-scaled if scaled < 0 else scaled)
if scaled < 0:
try self.writer.put("-")
try self.writer.put_u64(magnitude >> 7)
try self.writer.put(".")
let part = (magnitude & 127) * 78125
var digit: u64 = 1000000
while digit > 0:
try self.writer.put(decimal_digits[(usize)((part // digit) % 10)])
digit //= 10
## The function's literal pool (`float_constant`), eight bytes an entry, after its cold
## paths: a single's bits in the low half, which a little-endian `ldr s` reads.
func finish_pool() -> !:
if self.pool.count == 0:
return
try self.writer.put("\n .p2align 3")
for k in 0..<self.pool.count:
try self.writer.put("\n")
try self.pool_label(u32(k))
try self.writer.put(":\n .quad ")
try self.writer.put_u64(self.pool.get(k))
try self.writer.put("\n")
self.pool.count = 0
## The most instructions a function may have for a megabyte to cross it, the reach of a
## literal load and of a conditional branch: no instruction of the IR selects to more than
## sixty-four bytes. Past it, float constants are built in place (`float_constant`) and a
## conditional branch skips over an unconditional one (`branch_start`).
let short_reach: usize = 16384
## The value of a float of `bits`, double when `wide`, times 128 and signed, when `fmov`
## encodes it as an immediate: ±(16 + m)/16 · 2^e for a four-bit m and e in -3..4 (the
## architecture's VFPExpandImm), so the value times 128 is a whole number; none for any
## other value.
func fmov_scaled(bits: u64, wide: bool) -> i64?:
let fraction_bits: u64 = 52 if wide else 23
let bias: u64 = 1023 if wide else 127
let exponent_mask: u64 = 0x7FF if wide else 0xFF
if (bits & (((u64)1 << (fraction_bits - 4)) - 1)) != 0 or (not wide and bits > 0xFFFFFFFF):
return none
let exponent = (bits >> fraction_bits) & exponent_mask
if exponent + 3 < bias or exponent > bias + 4:
return none
let mantissa = (bits >> (fraction_bits - 4)) & 15
let scaled = (i64)((16 + mantissa) << (exponent + 3 - bias))
return -scaled if ((bits >> (63 if wide else 31)) & 1) != 0 else scaled
## The decimal digits, for an immediate spelled digit by digit.
let decimal_digits: str[10] = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
## The register name of class `cls` for scratch number `n` (0, 1, 2).
func scratch_register(cls: ir.Class, n: u32) -> str:
if cls == ir.Class.w:
return "w9" if n == 0 else ("w10" if n == 1 else "w11")
if cls == ir.Class.l:
return "x9" if n == 0 else ("x10" if n == 1 else "x11")
if cls == ir.Class.s:
return "s16" if n == 0 else ("s17" if n == 1 else "s18")
if cls == ir.Class.h:
return "h16" if n == 0 else ("h17" if n == 1 else "h18")
return "d16" if n == 0 else ("d17" if n == 1 else "d18")
## `w9` -> `x9`; a 64-bit immediate load wants the wide name.
func as_x(reg: str) -> str:
let b = reg.bytes
if b[0] == 'w':
if reg == "w9":
return "x9"
if reg == "w10":
return "x10"
if reg == "w11":
return "x11"
if reg == "w16":
return "x16"
if reg == "w0":
return "x0"
if reg == "w1":
return "x1"
if reg == "w2":
return "x2"
if reg == "w3":
return "x3"
if reg == "w4":
return "x4"
if reg == "w5":
return "x5"
if reg == "w6":
return "x6"
if reg == "w7":
return "x7"
return reg