dymokomi/luce-base
Source of release 0.46.0
0.46.0 / src / back / native / arm64 / atomics.lucb
#==============================================================================================
#
# arm64/atomics - Atomics
#
# DESCRIPTION:
# Atomic read-modify-write operations and compare-and-swap, as the target's
# instructions or a loop that retries (ยง15).
#
#==============================================================================================
extend Generator:
# mark: Atomics ============================================================================
## `fresh = old` when `old` compares `cond` (`ge` or `le`) to `value`, else `value`: the
## new value of an atomic max or min.
func extreme(fresh: str, old: str, value: str, cond: str) -> !:
try self.writer.line("cmp ")
try self.writer.put(old)
try self.writer.put(", ")
try self.writer.put(value)
try self.writer.line("csel ")
try self.writer.put(fresh)
try self.writer.put(", ")
try self.writer.put(old)
try self.writer.put(", ")
try self.writer.put(value)
try self.writer.put(", ")
try self.writer.put(cond)
## Read-modify-write atomics as an exclusive-load, compute, exclusive-store loop that
## retries until the store holds; acquire and release follow the ordering asked for,
## and the old value is the result.
func read_modify_write(n: ir.Instr) -> !:
let cls = ir.Class.w if n.extra <= 4 else ir.Class.l
let value = scratch_register(cls, 1)
try self.operand(value, cls, n.a)
try self.address_operand("x17", n.b)
let acquire = n.extra2 == 2 or n.extra2 == 4 or n.extra2 == 5
let release = n.extra2 >= 3
try self.writer.put("\n1:")
try self.writer.line("ldaxr" if acquire else "ldxr")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" ")
try self.writer.put(scratch_register(cls, 0))
try self.writer.put(", [x17]")
let old = scratch_register(cls, 0)
let fresh = scratch_register(cls, 2)
match n.op:
.atomic_add:
try self.writer.line("add ")
.atomic_sub:
try self.writer.line("sub ")
.atomic_or:
try self.writer.line("orr ")
.atomic_and_not:
try self.writer.line("bic ")
.atomic_xor:
try self.writer.line("eor ")
.atomic_swap:
try self.writer.line("mov ")
.atomic_max:
try self.extreme(fresh, old, value, "ge")
.atomic_min:
try self.extreme(fresh, old, value, "le")
_:
try self.writer.line("mov ")
if n.op != ir.Op.atomic_max and n.op != ir.Op.atomic_min:
try self.writer.put(fresh)
try self.writer.put(", ")
if n.op != ir.Op.atomic_swap:
try self.writer.put(old)
try self.writer.put(", ")
try self.writer.put(value)
try self.writer.line("stlxr" if release else "stxr")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" w16, ")
try self.writer.put(fresh)
try self.writer.put(", [x17]")
try self.writer.line("cbnz w16, 1b")
try self.store_temp(old, n.dst)
## Compare-and-swap as an `ldaxr`/`stlxr` loop: the store is tried only while memory
## holds the expected value, and what memory held is the result.
func compare_and_swap(n: ir.Instr) -> !:
let cls = ir.Class.w if n.extra <= 4 else ir.Class.l
let expected = self.f.args.get((usize)n.b.value)
let desired = self.f.args.get((usize)n.b.value + 1)
try self.operand(scratch_register(cls, 1), cls, expected.value)
try self.operand(scratch_register(cls, 2), cls, desired.value)
try self.address_operand("x17", n.a)
try self.writer.put("\n1:")
try self.writer.line("ldaxr")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" ")
try self.writer.put(scratch_register(cls, 0))
try self.writer.put(", [x17]")
try self.writer.line("cmp ")
try self.writer.put(scratch_register(cls, 0))
try self.writer.put(", ")
try self.writer.put(scratch_register(cls, 1))
try self.writer.line("b.ne 2f")
try self.writer.line("stlxr")
try self.writer.put(width_suffix(n.extra))
try self.writer.put(" w16, ")
try self.writer.put(scratch_register(cls, 2))
try self.writer.put(", [x17]")
try self.writer.line("cbnz w16, 1b")
try self.writer.put("\n2:")
try self.writer.line("clrex")
try self.store_temp(scratch_register(cls, 0), n.dst)