Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.46.0

0.46.0 / src / back / native / x86_64 / functions.lucb

#==============================================================================================
#
#   x86_64/functions - Functions
#
#   DESCRIPTION:
#       Each function's prologue, parameters received under the convention, body, and
#       epilogue.
#
#==============================================================================================
extend Generator:
    # mark: Functions ==========================================================================

    ## Copy `size` bytes from (`origin`) to (`to`) with plain loads and stores: 16 at a
    ## time through the first float scratch register, then 8, 4, 2, 1 through rax.
    ## Unaligned access is fine on this target.
    func inline_copy(origin: str, to: str, size: u64) -> !:
        var done: u64 = 0
        let through = self.reg(ir.Class.v, 0)
        while size - done >= 16:
            try self.based_move("movups ", done, origin, ", ")
            try self.writer.put(through)
            try self.writer.line3("movups ", through, ", ")
            try self.writer.put_u64(done)
            try self.writer.put("(")
            try self.put2(to, ")")
            done += 16
        let widths: u64[4] = [8, 4, 2, 1]
        let loads: str[4] = ["movq ", "movl ", "movzwl ", "movzbl "]
        let load_regs: str[4] = [", %rax", ", %eax", ", %eax", ", %eax"]
        let stores: str[4] = ["movq %rax, ", "movl %eax, ", "movw %ax, ", "movb %al, "]
        var k: usize = 0
        while k < 4:
            if size - done >= widths[k]:
                try self.based_move(loads[k], done, origin, load_regs[k])
                try self.based_move(stores[k], done, to, "")
                done += widths[k]
            k += 1

    ## Zero `size` bytes at (`to`) the same way.
    func inline_zero(to: str, size: u64) -> !:
        var done: u64 = 0
        let through = self.reg(ir.Class.v, 0)
        if size >= 16:
            try self.writer.line4("pxor ", through, ", ", through)
        while size - done >= 16:
            try self.writer.line3("movups ", through, ", ")
            try self.writer.put_u64(done)
            try self.writer.put("(")
            try self.put2(to, ")")
            done += 16
        let widths: u64[4] = [8, 4, 2, 1]
        let stores: str[4] = ["movq $0, ", "movl $0, ", "movw $0, ", "movb $0, "]
        var k: usize = 0
        while k < 4:
            if size - done >= widths[k]:
                try self.based_move(stores[k], done, to, "")
                done += widths[k]
            k += 1

    ## `mnemonic offset(base)rest`.
    func based_move(mnemonic: str, offset: u64, base: str, rest: str) -> !:
        try self.writer.line(mnemonic)
        try self.writer.put_u64(offset)
        try self.writer.put("(")
        try self.writer.put(base)
        try self.writer.put(")")
        try self.writer.put(rest)

    ## `mnemonic` + offset + `rest`: one line of a copy.
    func offset_move(mnemonic: str, offset: u64, rest: str) -> !:
        try self.writer.line(mnemonic)
        try self.writer.put_u64(offset)
        try self.writer.put(rest)

    ## Whether the function needs the third float scratch register (`reg(cls, 2)`): a fused
    ## multiply-add, a checked vector sum or difference, or an aggregate passed or returned
    ## whose last eightbyte may go between memory and an SSE register in part
    ## (`load_eightbyte`).
    func third_scratch() -> bool:
        if self.f.has_result and partial_eightbyte(self.f.result_size):
            return true
        for n in self.f.instrs.span():
            if n.op == ir.Op.fma or n.op == ir.Op.vaddo or n.op == ir.Op.vsubo:
                return true
            if n.op == ir.Op.call:
                let info = self.f.calls.get((usize)n.extra)
                if info.has_result and partial_eightbyte(info.result_size):
                    return true
                for k in 0..<info.args_count:
                    if partial_eightbyte(self.f.args.get((usize)(info.args_start + k)).size):
                        return true
        return false

    ## The instructions that overwrite rdx or rcx, temporaries' registers, themselves
    ## (`frame.Frame.fixed`): a division (rdx:rax), an unsigned checked multiply by other
    ## than a constant (rdx:rax), a shift by a variable count (cl), an atomic operation
    ## retried in a loop (rdx), and below x86-64-v3 a half's conversion through the unit's
    ## routines, which use both. No temporary lives in those across them; their operands
    ## and results may, and each sequence moves them out of the way.
    func fixed_clobbers() -> !:
        self.fr.fixed.count = 0
        let rdx = (u64)1 << (u64)(fixed_code(self.convention.windows, false) - 1)
        let rcx = (u64)1 << (u64)(fixed_code(self.convention.windows, true) - 1)
        for i in 0..<self.f.instrs.count:
            let n = self.f.instrs.get(i)
            var codes: u64 = 0
            match n.op:
                .sdiv, .udiv, .srem, .urem:
                    if is_int_class(n.cls):
                        codes = rdx
                .shl, .lshr, .ashr:
                    if is_int_class(n.cls) and not fits_immediate(n.b):
                        codes = rcx
                .mulo:
                    if n.extra != 1 and not constant_factor(n.b):
                        codes = rdx
                .atomic_or, .atomic_xor, .atomic_and_not, .atomic_max, .atomic_min:
                    codes = rdx
                _:
                    _ = 0
            if self.level < 3 and (n.cls == ir.Class.h or n.op == ir.Op.exth):
                codes = codes | rdx | rcx
            if codes != 0:
                try self.fr.fixed.push(regalloc.Clobber(at = u32(i), codes = codes))

    ## A load from a promoted slot: the register, extended as the load would extend from
    ## memory, since a narrow store leaves the register's upper bits as they were.
    func load_promoted(op: ir.Op, k: u32, d: str) -> !:
        let code = self.fr.slot_regs.get((usize)k - 1)
        let q = self.q_name(code)
        let l = self.l_name(code)
        match op:
            .loadl:
                if d != q:
                    try self.writer.line4("movq ", q, ", ", d)
            .loaduw:
                if d != l:
                    try self.writer.line4("movl ", l, ", ", d)
            .loadsw: try self.writer.line4("movslq ", l, ", ", d)
            .loadub: try self.writer.line4("movzbl ", self.b_name(code), ", ", d)
            .loadsb: try self.writer.line4("movsbl ", self.b_name(code), ", ", d)
            .loaduh: try self.writer.line4("movzwl ", self.w_name(code), ", ", d)
            .loadsh: try self.writer.line4("movswl ", self.w_name(code), ", ", d)
            _:
                error(failed, "a float load from a promoted slot")

    ## A store into a promoted slot: the value's register `v` moves in.
    func store_promoted(op: ir.Op, k: u32, v: str) -> !:
        let code = self.fr.slot_regs.get((usize)k - 1)
        match op:
            .storel:
                if v != self.q_name(code):
                    try self.writer.line4("movq ", v, ", ", self.q_name(code))
            .storew:
                if v != self.l_name(code):
                    try self.writer.line4("movl ", v, ", ", self.l_name(code))
            .storeh: try self.writer.line4("movw ", word_name(v), ", ", self.w_name(code))
            .storeb: try self.writer.line4("movb ", byte_name(v), ", ", self.b_name(code))
            _:
                error(failed, "a float store into a promoted slot")

    ## Emit the instructions from `i` that select as one, when they do (a comparison and
    ## its branch, an address and its access): how many they were, or 0 when the one at `i`
    ## selects alone.
    func fused_at(i: usize, uses: u32[], index: usize) -> usize!:
        if selection.branch(self.f, i, uses, true):
            try self.branch_pair(i, index)
            return 2
        if selection.compared_select(self.f, i, uses):
            try self.mark_at(index, i + 1)
            try self.select_on(self.f.instrs.get(i), self.f.instrs.get(i + 1))
            return 2
        if let fused = selection.offset_access(self.f, i, uses):
            let (base, offset) = fused
            if not self.fr.is_rematerialised(self.f.instrs.get(i).dst):
                if let dw = self.dw:
                    try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
                let (access, widened) = self.access_at(i + 1, uses)
                try self.mark_widening(index, i + 1, widened)
                try self.offset_access(access, base, offset)
                return 2 + widened
        if let scaled = selection.scaled_access(self.f, i, uses):
            let (base, scaled_index, shift) = scaled
            if not self.fr.is_rematerialised(self.f.instrs.get(i).dst) and not self.fr.is_rematerialised(self.f.instrs.get(i + 1).dst):
                if let dw = self.dw:
                    try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 2)
                let (access, widened) = self.access_at(i + 2, uses)
                try self.mark_widening(index, i + 2, widened)
                try self.indexed_access(access, base, scaled_index, shift)
                return 3 + widened
        if let summed = selection.summed_access(self.f, i, uses):
            let (base, offset) = summed
            if not self.fr.is_rematerialised(self.f.instrs.get(i).dst):
                if let dw = self.dw:
                    try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
                let (access, widened) = self.access_at(i + 1, uses)
                try self.mark_widening(index, i + 1, widened)
                try self.indexed_access(access, base, offset, 0)
                return 2 + widened
        if let load = selection.widened_load(self.f, i, uses):
            try self.mark_at(index, i + 1)
            try self.instruction(load, selection.label_at(self.f, i + 2))
            return 2
        return 0

    ## The debug labels and lines of instruction `at`, which a fused selection consumes: a
    ## scope may begin or end there, so its label must exist.
    func mark_at(index: usize, at: usize) -> !:
        if let dw = self.dw:
            try dw.mark(&self.writer.out, self.f, &self.fr, index, at)

    ## The same for the widening after the access at `at`, when the access took it as well.
    func mark_widening(index: usize, at: usize, widened: usize) -> !:
        if widened != 0:
            try self.mark_at(index, at + 1)

    ## The load or store at `at` a fused address reaches, and 1 when it is a signed word's
    ## load whose widening after it it does as well (`selection.widened_load`), else 0.
    func access_at(at: usize, uses: u32[]) -> (ir.Instr, usize):
        if let load = selection.widened_load(self.f, at, uses):
            return (load, 1)
        return (self.f.instrs.get(at), 0)

    ## The comparison at `i` and the branch on it after, as one conditional jump.
    func branch_pair(i: usize, index: usize) -> !:
        if let dw = self.dw:
            # the fused branch's own line and labels, ahead of the pair
            try dw.mark(&self.writer.out, self.f, &self.fr, index, i + 1)
        try self.fused_branch(i, selection.label_at(self.f, i + 2))

    ## The comparison at `i` and the branch on it after, with `next` as the fall-through:
    ## the pair in place, or a rotated loop's head test at a jump to that head
    ## (`selection.rotations`).
    func fused_branch(i: usize, next: u32) -> !:
        let branch = self.f.instrs.get(i + 1)
        var comparison = self.f.instrs.get(i)
        if ir.is_float_class(comparison.cls):
            # a float comparison's negation is no comparison of its own: unordered operands
            # fail both
            try self.float_branch(comparison, branch, next)
            return
        if next == branch.extra:
            comparison.op = ir.negated(comparison.op)
        try self.comparison(comparison, branch, next)

    ## `.weak` for a weak symbol (§9.8); ELF has no directive for `used`, which only matters
    ## to a linker that discards sections, so it is left to the section's own flags.
    func function(index: usize) -> !:
        self.index = index
        if let dw = self.dw:
            try dw.entry(&self.writer.out)
        self.f = &self.program_unit.functions.items[index]
        let start = self.writer.out.used
        try self.function_text(index, none)
        # Windows: the save area holds just the nonvolatile registers the text names; when
        # the allocator's prediction of them missed, the function is laid out again with
        # those (its registers do not depend on the area, so the second text names the same)
        if self.convention.windows and not self.f.naked and self.dw == none and self.windows_used != self.windows_plan:
            self.writer.out.used = start
            try self.function_text(index, self.windows_used)
            if self.windows_used != self.windows_plan:
                error(failed, "the Windows nonvolatile registers a function names changed when it was laid out again")

    ## Function `index`'s text; on Windows with its save area holding `plan`, or the
    ## registers the allocator gave it (a debug build's, all of them) when none.
    func function_text(index: usize, plan: u16?) -> !:
        self.fr.reserved = 0
        self.pool.count = 0
        if self.third_scratch():
            self.fr.reserve(third_scratch_code(self.convention.windows))
        self.reserve_asm_registers()
        try layout.arrange(self.f, false)
        try selection.sink_addresses(self.f)
        try self.fixed_clobbers()
        try self.fr.plan(self.f)
        var threaded = try layout.thread_after_allocation(self.f, &self.fr)
        defer threaded.restore(self.f)
        self.windows_plan = 0
        if self.convention.windows:
            self.windows_plan = plan else (4095 if self.dw != none else self.allocated_saves())
        self.crossing = 0
        var entry = try selection.entry_loads(self.f)
        defer free(entry)
        let straight = try self.straight_params(entry)
        defer free(straight)
        try self.fr.layout(self.f, save_area(self.windows_plan), self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private), straight)
        let text_start = self.writer.out.used
        try self.object_format.function_start(&self.writer.out, self.f)
        if self.f.naked:
            # the body is asm blocks only (§9.8): no frame, no prologue, nothing after, not
            # even the `ret` the lowerer ends every function with
            var k: usize = 0
            while k < self.f.instrs.count:
                self.at = k
                if self.f.instrs.get(k).op == ir.Op.asm_:
                    try self.instruction(self.f.instrs.get(k))
                k += 1
            try self.object_format.function_end(&self.writer.out, self.f)
            return
        try self.unwinding.begin(&self.writer.out, self.symbol_prefix, self.f.name)
        try self.writer.line("pushq %rbp")
        try self.unwinding.pushed_frame_pointer(&self.writer.out)
        if not self.convention.windows:
            try self.writer.line("movq %rsp, %rbp")
        try self.unwinding.cfa_frame_pointer(&self.writer.out)
        if self.convention.windows and self.fr.size >= 4096:
            try self.load_immediate("%rax", (i64)self.fr.size)
            try self.writer.line("call ___chkstk_ms")
        if self.fr.size > 0:
            try self.writer.line("subq $")
            try self.writer.put_u64(self.fr.size)
            try self.writer.put(", %rsp")
            try self.unwinding.allocated_stack(&self.writer.out, self.fr.size)
        if self.convention.windows:
            try self.writer.line("movq %rsp, %rbp")
            try self.unwinding.established_frame(&self.writer.out, self.fr.size)
        if self.f.has_result and self.result_in_memory(self.f.result_size, self.f.result_shape, self.f.result_private):
            try self.writer.line("movq %rcx, " if self.convention.windows else "movq %rdi, ")
            try self.frame_operand(self.fr.indirect_home)
        try self.save_callee_saved()
        try self.unwinding.prologue_complete(&self.writer.out)
        var moves = try list.List[incoming.Move].create(8)
        defer moves.destroy()
        try self.receive_params(entry, &moves)
        var uses = try selection.uses(self.f)
        defer free(uses)
        var received = try self.receive_directly(&moves, uses)
        defer free(received)
        var dynamic = try levels.over(self.f)
        defer dynamic.release()
        var heads = try selection.loop_heads(self.f)
        defer free(heads)
        var tests = try selection.rotations(self.f, heads, uses, false)
        defer free(tests)
        selection.rotate_heads(self.f, heads, tests)
        self.loop_heads = heads
        self.head_tests = tests
        var i: usize = 0
        while i < self.f.instrs.count:
            self.at = i
            if let dw = self.dw:
                try dw.mark(&self.writer.out, self.f, &self.fr, index, i)
            if received[i]:
                i += 1
                continue
            let fused = try self.fused_at(i, uses, index)
            if fused > 0:
                i += fused
                continue
            try self.save_crossing(i)
            try self.instruction(self.f.instrs.get(i), selection.label_at(self.f, i + 1))
            try self.restore_crossing(i)
            if dynamic.settle[i]:
                # edges bring different levels: %rsp goes below the most of them
                try self.reserve_levels(dynamic.before[i + 1])
            i += 1
        self.loop_heads = heads[0..<0]
        self.head_tests = tests[0..<0]
        try self.finish_function(index, i, text_start)

    ## After function `index`'s last instruction (`end`): its debug information, unwinding
    ## and Windows saves (its text began at `text_start`), then out of its section.
    func finish_function(index: usize, end: usize, text_start: usize) -> !:
        try self.finish_cold()
        if let dw = self.dw:
            try dw.mark(&self.writer.out, self.f, &self.fr, index, end)
            if self.f.debug or self.f.symbols:
                try self.writer.put("\nLdw_end_")
                try self.writer.put_u64((u64)index)
                try self.writer.put(":\n")
            try self.unwinding.cfi_complete(&self.writer.out)
            try dw.function(self.f, &self.fr, index)
        try self.unwinding.complete(&self.writer.out)
        try self.writer.put("\n")
        try self.settle_windows_saves(text_start)
        if self.f.debug:
            try assembly.describe_frame(&self.descriptors, self.symbol_prefix, ".Ltext_", self.program_unit, self.f, &self.fr)
        try self.object_format.function_end(&self.writer.out, self.f)
        try self.finish_pool()