dymokomi/luce-base
Source of release 0.42.0
0.42.0 / src / back / native / arm64 / vectors.lucb
#==============================================================================================
#
# arm64/vectors - Vectors
#
# DESCRIPTION:
# Lane-wise vector operations (§5.12).
#
#==============================================================================================
extend Generator:
# mark: Vectors (§5.12) ====================================================================
## A vector instruction on NEON: the operands come from their homes into v16 and v17,
## the result computes into v16 (v18 for a checked form) and goes to its home. The lane
## shape picks the arrangement, `.4s` or `.8b`; an eight-byte vector is the low half
## of the register, loaded and stored as `d`.
func vector(n: ir.Instr) -> !:
let shape = n.extra
let bytes = ir.vector_bytes(shape)
let arr = arrangement(shape)
let all = ".16b" if bytes == 16 else ".8b"
# Full-width operations can read and write their allocated registers
# directly. Narrow and checked vectors keep the specialized sequences.
if (bytes == 16 or ir.lanes_float(shape)) and try self.full_vector(n, arr, all):
return
match n.op:
.loadv:
try self.address_operand("x9", n.a)
try self.writer.line3("ldr ", vector_scratch(0, bytes), ", [x9]")
try self.vector_result(n.dst, 0, bytes)
.storev:
try self.vector_source(0, n.a, bytes)
try self.address_operand("x9", n.b)
try self.writer.line3("str ", vector_scratch(0, bytes), ", [x9]")
.vsplat:
try self.splat(n, shape, arr)
try self.vector_result(n.dst, 0, bytes)
.vneg, .vnot:
try self.vector_source(0, n.a, bytes)
let form = arr if n.op == ir.Op.vneg else all
try self.writer.line("fneg v16" if n.op == ir.Op.vneg else "mvn v16")
try self.writer.put(form)
try self.writer.put(", v16")
try self.writer.put(form)
try self.vector_result(n.dst, 0, bytes)
.vshl, .vshr:
try self.vector_shift(n, arr, bytes)
.vaddo, .vsubo:
try self.vector_checked(n, arr, all, bytes)
_:
try self.vector_source(0, n.a, bytes)
try self.vector_source(1, n.b, bytes)
let bitwise = n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor
let form = all if bitwise else arr
try self.writer.line(vector_mnemonic(n.op, ir.lanes_signed(shape)))
try self.writer.put(" v16")
try self.writer.put(form)
try self.writer.put(", v16")
try self.writer.put(form)
try self.writer.put(", v17")
try self.writer.put(form)
try self.vector_result(n.dst, 0, bytes)
## A sixteen-byte load, store, or lane-wise arithmetic straight between the allocated
## registers; false for an operation that takes the scratch sequences instead.
func full_vector(n: ir.Instr, arr: str, all: str) -> bool!:
if n.op == ir.Op.vsplat and ir.lanes_float(n.extra):
# a float straight from where it lives into every lane of the result's register
let cls = ir.Class.d if ir.lane_bits(n.extra) == 64 else ir.Class.s
let v = try self.source(n.a, cls, scratch_register(cls, 0))
let d = self.target(n.dst, ir.Class.v, "q16")
try self.writer.line4("dup ", lanes_view(d), arr, ", ")
try self.writer.put(lanes_view(v))
try self.writer.put(".d[0]" if cls == ir.Class.d else ".s[0]")
try self.finish(n.dst, ir.Class.v, d)
return true
# an eight-byte vector is the register's low half, loaded and stored as `d`
let wide = ir.vector_bytes(n.extra) == 16
if n.op == ir.Op.loadv:
let address = try self.source(n.a, ir.Class.l, "x9")
let d = self.target(n.dst, ir.Class.v, "q16")
try self.writer.line4("ldr ", d if wide else scalar_view(d, ir.Class.d), ", [", address)
try self.writer.put("]")
try self.finish(n.dst, ir.Class.v, d)
return true
if n.op == ir.Op.storev:
let value = try self.source(n.a, ir.Class.v, "q16")
let address = try self.source(n.b, ir.Class.l, "x9")
try self.writer.line4("str ", value if wide else scalar_view(value, ir.Class.d), ", [", address)
try self.writer.put("]")
return true
if n.op == ir.Op.vadd or n.op == ir.Op.vsub or n.op == ir.Op.vmul or n.op == ir.Op.vdiv or n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor or n.op == ir.Op.vaddw or n.op == ir.Op.vsubw or n.op == ir.Op.vmulw:
let a = try self.source(n.a, ir.Class.v, "q16")
let b = try self.source(n.b, ir.Class.v, "q17")
let d = self.target(n.dst, ir.Class.v, "q16")
let form = all if n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor else arr
try self.writer.line(vector_mnemonic(n.op, ir.lanes_signed(n.extra)))
try self.writer.put(" ")
try self.writer.put(vector_view(d))
try self.writer.put(form)
try self.writer.put(", ")
try self.writer.put(vector_view(a))
try self.writer.put(form)
try self.writer.put(", ")
try self.writer.put(vector_view(b))
try self.writer.put(form)
try self.finish(n.dst, ir.Class.v, d)
return true
return false
## The lanes the optimiser packs from scalars (`opt/slp`), between the allocated
## registers: a scalar into a lane (`ins`), a lane out to a scalar (lane 0 is the
## register's own scalar view), and the square root of every lane. An inserted scalar
## sharing the result's register is moved aside before the vector it joins is copied in.
func packed_lanes(n: ir.Instr) -> !:
let shape = n.extra
let wide = ir.lane_bits(shape) == 64
let cls = ir.Class.d if wide else ir.Class.s
let lane = ".d[" if wide else ".s["
if n.op == ir.Op.vsqrt or n.op == ir.Op.vexts or n.op == ir.Op.vtruncd:
# the source's lanes: as many as the result's, of the other width when converting
var source_lanes = arrangement(shape)
var mnemonic = "fsqrt "
if n.op == ir.Op.vexts:
source_lanes = ".2s"
mnemonic = "fcvtl "
elif n.op == ir.Op.vtruncd:
source_lanes = ".2d"
mnemonic = "fcvtn "
let a = try self.source(n.a, ir.Class.v, "q16")
let d = self.target(n.dst, ir.Class.v, "q16")
try self.writer.line4(mnemonic, lanes_view(d), arrangement(shape), ", ")
try self.writer.put(lanes_view(a))
try self.writer.put(source_lanes)
try self.finish(n.dst, ir.Class.v, d)
return
if n.op == ir.Op.vextract:
let a = try self.source(n.a, ir.Class.v, "q16")
let d = self.target(n.dst, cls, scratch_register(cls, 0))
if n.extra2 == 0:
let low = scalar_view(a, cls)
if low != d:
try self.fmove(d, low)
else:
try self.writer.line4("mov ", d, ", ", lanes_view(a))
try self.writer.put(lane)
try self.writer.put_u64((u64)n.extra2)
try self.writer.put("]")
try self.finish(n.dst, cls, d)
return
var b = try self.source(n.b, cls, scratch_register(cls, 1))
let d = self.target(n.dst, ir.Class.v, "q16")
if n.a.kind == ir.RefKind.none_ and n.extra2 == 0:
# the other lanes are anything: the scalar's own move
if lanes_view(b) != lanes_view(d):
try self.fmove(scalar_view(d, cls), b)
try self.finish(n.dst, ir.Class.v, d)
return
if lanes_view(b) == lanes_view(d):
try self.fmove(scratch_register(cls, 1), b)
b = scratch_register(cls, 1)
if n.a.kind != ir.RefKind.none_:
let a = try self.source(n.a, ir.Class.v, "q16")
if lanes_view(a) != lanes_view(d):
try self.writer.line4("mov ", lanes_view(d), ".16b, ", lanes_view(a))
try self.writer.put(".16b")
try self.writer.line4("mov ", lanes_view(d), lane, "")
try self.writer.put_u64((u64)n.extra2)
try self.writer.put("], ")
try self.writer.put(lanes_view(b))
try self.writer.put(lane)
try self.writer.put("0]")
try self.finish(n.dst, ir.Class.v, d)
## `v << count` and `v >> count`: the count, one scalar, spread over the lanes; a right
## shift is a left shift by the negated count, arithmetic or logical by the lanes'
## signedness.
func vector_shift(n: ir.Instr, arr: str, bytes: u32) -> !:
let shape = n.extra
try self.vector_source(0, n.a, bytes)
let wide = ir.lane_bits(shape) == 64
let count = try self.source(n.b, ir.Class.l if wide else ir.Class.w, "x9" if wide else "w9")
try self.writer.line("dup v17")
try self.writer.put(arr)
try self.writer.put(", ")
try self.writer.put(count)
if n.op == ir.Op.vshr:
try self.writer.line("neg v17")
try self.writer.put(arr)
try self.writer.put(", v17")
try self.writer.put(arr)
try self.writer.line("sshl v16" if ir.lanes_signed(shape) else "ushl v16")
try self.writer.put(arr)
try self.writer.put(", v16")
try self.writer.put(arr)
try self.writer.put(", v17")
try self.writer.put(arr)
try self.vector_result(n.dst, 0, bytes)
## Checked `+` and `-` over lanes: the wrapping result beside the saturating one; a lane
## that differs overflowed, and traps.
func vector_checked(n: ir.Instr, arr: str, all: str, bytes: u32) -> !:
try self.vector_source(0, n.a, bytes)
try self.vector_source(1, n.b, bytes)
let add = n.op == ir.Op.vaddo
try self.writer.line("add v18" if add else "sub v18")
try self.writer.put(arr)
try self.writer.put(", v16")
try self.writer.put(arr)
try self.writer.put(", v17")
try self.writer.put(arr)
let signed = ir.lanes_signed(n.extra)
try self.writer.line(("sqadd v16" if signed else "uqadd v16") if add else ("sqsub v16" if signed else "uqsub v16"))
try self.writer.put(arr)
try self.writer.put(", v16")
try self.writer.put(arr)
try self.writer.put(", v17")
try self.writer.put(arr)
try self.writer.line("cmeq v16")
try self.writer.put(arr)
try self.writer.put(", v16")
try self.writer.put(arr)
try self.writer.put(", v18")
try self.writer.put(arr)
try self.writer.line("uminv b16, v16")
try self.writer.put(all)
try self.writer.line("fmov w9, s16")
try self.writer.line("cbnz w9, 1f")
try self.trap_text("integer overflow", n.extra2)
try self.writer.put("\n1:")
try self.vector_result(n.dst, 2, bytes)
## `T[N](x)`: the scalar into every lane of v16.
func splat(n: ir.Instr, shape: u32, arr: str) -> !:
if ir.lanes_float(shape):
let cls = ir.Class.d if ir.lane_bits(shape) == 64 else ir.Class.s
try self.operand(scratch_register(cls, 0), cls, n.a)
try self.writer.line("dup v16")
try self.writer.put(arr)
try self.writer.put(", v16.d[0]" if cls == ir.Class.d else ", v16.s[0]")
return
let wide = ir.lane_bits(shape) == 64
let scalar = try self.source(n.a, ir.Class.l if wide else ir.Class.w, "x9" if wide else "w9")
try self.writer.line("dup v16")
try self.writer.put(arr)
try self.writer.put(", ")
try self.writer.put(scalar)
## A vector temporary into scratch register `k`: from the register it lives in, else
## from its home.
func vector_source(k: u32, r: ir.Ref, bytes: u32) -> !:
try self.load_temp(vector_scratch(k, bytes), ir.Class.v, r.index)
## Scratch register `k` into vector temporary `dst`, where it lives.
func vector_result(dst: u32, k: u32, bytes: u32) -> !:
if dst == 0:
return
try self.store_temp(vector_scratch(k, bytes), dst)