dymokomi/luce-base
Source of release 0.45.0
0.45.0 / src / back / native / x86_64 / vectors.lucb
#==============================================================================================
#
# x86_64/vectors - Vectors
#
# DESCRIPTION:
# Lane-wise vector operations (§5.12).
#
#==============================================================================================
extend Generator:
# mark: Vectors (§5.12) ====================================================================
## A vector instruction on SSE2: the operands come from their homes into the first two
## scratch registers (`reg`), the result computes into the first (the third for a
## checked form) and goes to its home. An eight-byte vector is the low half of the
## register, moved as a quadword; the upper lanes then hold zeros, which no instruction
## here reads.
func vector(n: ir.Instr) -> !:
let shape = n.extra
let bytes = ir.vector_bytes(shape)
let bits = ir.lane_bits(shape)
let signed = ir.lanes_signed(shape)
let first = self.reg(ir.Class.v, 0)
let second = self.reg(ir.Class.v, 1)
if try self.float_lanes(n, first):
return
if bytes == 16:
if n.op == ir.Op.loadv:
try self.address_operand("%r10", n.a)
let d = self.target(n.dst, ir.Class.v, first)
# float lanes move in the float domain: no bypass delay before an FP operation
try self.writer.line3("movups (%r10), " if ir.lanes_float(shape) else "movdqu (%r10), ", d, "")
try self.finish(n.dst, ir.Class.v, d)
return
if n.op == ir.Op.storev:
let value = try self.source(n.a, ir.Class.v, first)
try self.address_operand("%r10", n.b)
try self.writer.line3("movups " if ir.lanes_float(shape) else "movdqu ", value, ", (%r10)")
return
if n.op == ir.Op.vadd or n.op == ir.Op.vsub or n.op == ir.Op.vmul or n.op == ir.Op.vdiv or n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor or n.op == ir.Op.vaddw or n.op == ir.Op.vsubw or n.op == ir.Op.vmulw:
let a = try self.source(n.a, ir.Class.v, first)
let b = try self.source(n.b, ir.Class.v, second)
var d = self.target(n.dst, ir.Class.v, first)
# SSE uses a destructive destination. Preserve a distinct right
# operand when the allocator gave its register to the result.
if d == b and d != a:
d = first
if a != d:
try self.writer.line4("movaps " if ir.lanes_float(shape) else "movdqa ", a, ", ", d)
let mnemonic = "pmulld" if n.op == ir.Op.vmulw and bits == 32 else vector_mnemonic(n.op, shape)
try self.writer.line4(mnemonic, " ", b, ", ")
try self.writer.put(d)
try self.finish(n.dst, ir.Class.v, d)
return
match n.op:
.loadv:
try self.address_operand("%r10", n.a)
try self.writer.line3(vector_move(bytes), "(%r10), ", first)
try self.vector_result(n.dst, first, bytes)
.storev:
try self.vector_source(n.a, first, bytes)
try self.address_operand("%r10", n.b)
try self.writer.line4(vector_move(bytes), first, ", ", "(%r10)")
.vsplat:
try self.splat(n, shape, first)
try self.vector_result(n.dst, first, bytes)
.vneg:
# the sign bit of every lane flipped
try self.vector_source(n.a, first, bytes)
try self.writer.line4("xorps ", self.shared(".Lvsign64" if bits == 64 else ".Lvsign32"), "(%rip), ", first)
try self.vector_result(n.dst, first, bytes)
.vnot:
try self.vector_source(n.a, first, bytes)
try self.writer.line4("pcmpeqd ", second, ", ", second)
try self.writer.line4("pxor ", second, ", ", first)
try self.vector_result(n.dst, first, bytes)
.vshl, .vshr:
# the count, one scalar, in the second scratch register's low quadword
try self.vector_source(n.a, first, bytes)
try self.operand("%rax", ir.Class.l, n.b)
try self.writer.line3("movq %rax, ", second, "")
var mnemonic = "psllw"
if n.op == ir.Op.vshl:
mnemonic = "psllw" if bits == 16 else ("pslld" if bits == 32 else "psllq")
elif signed:
mnemonic = "psraw" if bits == 16 else "psrad"
else:
mnemonic = "psrlw" if bits == 16 else ("psrld" if bits == 32 else "psrlq")
try self.writer.line5(mnemonic, " ", second, ", ", first)
try self.vector_result(n.dst, first, bytes)
.vaddo, .vsubo:
try self.checked_lanes(n, shape)
_:
try self.vector_source(n.a, first, bytes)
try self.vector_source(n.b, second, bytes)
if n.op == ir.Op.vmulw and bits == 32:
# x86-64-v2: the doubleword multiply of SSE4.1
try self.writer.line4("pmulld ", second, ", ", first)
else:
try self.writer.line5(vector_mnemonic(n.op, shape), " ", second, ", ", first)
try self.vector_result(n.dst, first, bytes)
## Float lanes straight between their registers: two singles to and from memory as the
## register's low quadword, and a float broadcast into every lane; false for anything
## else.
func float_lanes(n: ir.Instr, first: str) -> bool!:
let shape = n.extra
let bytes = ir.vector_bytes(shape)
let bits = ir.lane_bits(shape)
if not ir.lanes_float(shape):
return false
if bytes == 8 and (n.op == ir.Op.loadv or n.op == ir.Op.storev):
# two singles, the register's low quadword, between their registers and memory
try self.address_operand("%r10", n.b if n.op == ir.Op.storev else n.a)
if n.op == ir.Op.loadv:
let d = self.target(n.dst, ir.Class.v, first)
try self.writer.line3("movsd (%r10), ", d, "")
try self.finish(n.dst, ir.Class.v, d)
else:
let value = try self.source(n.a, ir.Class.v, first)
try self.writer.line3("movsd ", value, ", (%r10)")
return true
if bytes == 16 and n.op == ir.Op.vsplat:
# a float straight from where it lives into every lane of the result's register
let cls = ir.Class.d if bits == 64 else ir.Class.s
let v = try self.source(n.a, cls, self.reg(cls, 1))
let d = self.target(n.dst, ir.Class.v, first)
if v != d:
try self.writer.line4("movaps ", v, ", ", d)
try self.writer.line4("unpcklpd " if bits == 64 else "shufps $0, ", d, ", ", d)
try self.finish(n.dst, ir.Class.v, d)
return true
return false
## The lanes the optimiser packs from scalars (`opt/slp`), between the allocated
## registers: a scalar into lane 0 (`movsd`/`movss`, merging) or lane 1 (`unpcklpd`, or
## `unpcklps` for a two-lane single vector, the only shapes the optimiser inserts
## into), a lane out to a scalar (lane 0 is the register itself; `unpckhpd` or `shufps`
## bring another down), and the square root of every lane. An inserted scalar sharing
## the result's register is moved aside before the vector it joins is copied in.
func packed_lanes(n: ir.Instr) -> !:
let shape = n.extra
let wide = ir.lane_bits(shape) == 64
let cls = ir.Class.d if wide else ir.Class.s
if n.op == ir.Op.vsqrt or n.op == ir.Op.vexts or n.op == ir.Op.vtruncd:
let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
let d = self.target(n.dst, ir.Class.v, self.reg(ir.Class.v, 0))
var mnemonic = "sqrtpd " if wide else "sqrtps "
if n.op == ir.Op.vexts:
mnemonic = "cvtps2pd "
elif n.op == ir.Op.vtruncd:
mnemonic = "cvtpd2ps "
try self.writer.line4(mnemonic, a, ", ", d)
try self.finish(n.dst, ir.Class.v, d)
return
if n.op == ir.Op.vextract:
let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
let d = self.target(n.dst, cls, self.reg(cls, 0))
if a != d:
try self.writer.line4("movaps ", a, ", ", d)
if n.extra2 > 0 and wide:
try self.writer.line4("unpckhpd ", d, ", ", d)
elif n.extra2 > 0:
try self.writer.line("shufps $")
try self.writer.put_u64((u64)n.extra2)
try self.put2(", ", d)
try self.put2(", ", d)
try self.finish(n.dst, cls, d)
return
var b = try self.source(n.b, cls, self.reg(cls, 1))
let d = self.target(n.dst, ir.Class.v, self.reg(ir.Class.v, 0))
if n.a.kind == ir.RefKind.none_ and n.extra2 == 0:
# the other lanes are anything: the scalar's own move
if b != d:
try self.writer.line4("movaps ", b, ", ", d)
try self.finish(n.dst, ir.Class.v, d)
return
if b == d:
try self.writer.line4("movaps ", b, ", ", self.reg(cls, 1))
b = self.reg(cls, 1)
if n.a.kind != ir.RefKind.none_:
let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
if a != d:
try self.writer.line4("movaps ", a, ", ", d)
if n.extra2 == 0:
try self.writer.line4("movsd " if wide else "movss ", b, ", ", d)
else:
try self.writer.line4("unpcklpd " if wide else "unpcklps ", b, ", ", d)
try self.finish(n.dst, ir.Class.v, d)
## `vaddo`/`vsubo`: the wrapping result in the third scratch register; a lane that
## overflowed traps. With `a` and `b` in the first two: bytes and halfwords compare the
## wrapping result with the saturating one; doublewords read the sign of the overflow
## test, `(a ^ r) & (b ^ r)` for an add, `(a ^ b) & (a ^ r)` for a subtract, or the
## unsigned order of the operands through a sign flip. `a` and `b` are spent on it.
func checked_lanes(n: ir.Instr, shape: u32) -> !:
let bytes = ir.vector_bytes(shape)
let bits = ir.lane_bits(shape)
let signed = ir.lanes_signed(shape)
let add = n.op == ir.Op.vaddo
let a = self.reg(ir.Class.v, 0)
let b = self.reg(ir.Class.v, 1)
let r = self.reg(ir.Class.v, 2)
try self.vector_source(n.a, a, bytes)
try self.vector_source(n.b, b, bytes)
try self.writer.line4("movdqa ", a, ", ", r)
try self.writer.line5(lane_mnemonic("padd" if add else "psub", bits), " ", b, ", ", r)
if bits <= 16:
let sat = ("padds" if signed else "paddus") if add else ("psubs" if signed else "psubus")
try self.writer.line5(lane_mnemonic(sat, bits), " ", b, ", ", a)
try self.writer.line4("pcmpeqb ", r, ", ", a)
try self.writer.line3("pmovmskb ", a, ", %eax")
if bytes == 16:
try self.writer.line("cmpl $65535, %eax")
else:
try self.writer.line("andl $255, %eax")
try self.writer.line("cmpl $255, %eax")
try self.writer.line("je 1f")
elif signed:
# the overflow test's sign bits, into `a`
if add:
try self.writer.line4("pxor ", r, ", ", a)
try self.writer.line4("pxor ", r, ", ", b)
else:
try self.writer.line4("pxor ", a, ", ", b)
try self.writer.line4("pxor ", r, ", ", a)
try self.writer.line4("pand ", b, ", ", a)
try self.writer.line3("pmovmskb ", a, ", %eax")
try self.writer.line("andl $34952, %eax" if bytes == 16 else "andl $136, %eax")
try self.writer.line("je 1f")
else:
# unsigned: an add overflowed where r < a, a subtract where a < b; the sign
# flip makes the signed compare an unsigned one
let sign = self.shared(".Lvsign32")
try self.writer.line4("pxor ", sign, "(%rip), ", a)
if add:
# b = r with its signs flipped; a > b where r < a
try self.writer.line4("movdqa ", r, ", ", b)
try self.writer.line4("pxor ", sign, "(%rip), ", b)
try self.writer.line4("pcmpgtd ", b, ", ", a)
else:
# b > a where a < b
try self.writer.line4("pxor ", sign, "(%rip), ", b)
try self.writer.line4("pcmpgtd ", a, ", ", b)
try self.writer.line4("movdqa ", b, ", ", a)
try self.writer.line3("pmovmskb ", a, ", %eax")
try self.writer.line("andl $65535, %eax" if bytes == 16 else "andl $255, %eax")
try self.writer.line("je 1f")
try self.trap_text("integer overflow", n.extra2)
try self.writer.put("\n1:")
try self.vector_result(n.dst, r, bytes)
## `T[N](x)`: the scalar into every lane of `into`.
func splat(n: ir.Instr, shape: u32, into: str) -> !:
let bits = ir.lane_bits(shape)
if ir.lanes_float(shape):
if bits == 64:
try self.operand(into, ir.Class.d, n.a)
try self.writer.line4("unpcklpd ", into, ", ", into)
else:
try self.operand(into, ir.Class.s, n.a)
try self.writer.line4("shufps $0, ", into, ", ", into)
return
if bits == 64:
try self.operand("%rax", ir.Class.l, n.a)
try self.writer.line3("movq %rax, ", into, "")
try self.writer.line4("punpcklqdq ", into, ", ", into)
return
try self.operand("%eax", ir.Class.w, n.a)
try self.writer.line3("movd %eax, ", into, "")
if bits == 8:
try self.writer.line4("punpcklbw ", into, ", ", into)
if bits <= 16:
try self.writer.line4("punpcklwd ", into, ", ", into)
try self.writer.line4("pshufd $0, ", into, ", ", into)
## A vector temporary into `reg`: from the register it lives in, else from its home.
func vector_source(r: ir.Ref, reg: str, bytes: u32) -> !:
let code = self.fr.temp_regs.get((usize)r.index - 1)
if code != 0:
try self.writer.line4("movdqa ", self.class_name(code, ir.Class.v), ", ", reg)
return
try self.writer.line(vector_move(bytes))
try self.frame_operand(self.temp_home(r.index))
try self.writer.put(", ")
try self.writer.put(reg)
## `reg` into vector temporary `dst`, where it lives.
func vector_result(dst: u32, reg: str, bytes: u32) -> !:
if dst == 0:
return
let code = self.fr.temp_regs.get((usize)dst - 1)
if code != 0:
try self.writer.line4("movdqa ", reg, ", ", self.class_name(code, ir.Class.v))
return
try self.writer.line3(vector_move(bytes), reg, ", ")
try self.frame_operand(self.temp_home(dst))