dymokomi/luce-base
Source of release 0.45.0
0.45.0 / src / support / pieces.lucb
#==============================================================================================
#
# pieces - One program's assembly assembled in pieces, in parallel
#
# DESCRIPTION:
# The system assembler reads one file on one thread: luced-3d's 70 MB of assembly
# took it ten seconds, and a small program importing a large package spent most of a
# rebuild there. Generated with `split`, the assembly names the labels functions share
# as hidden symbols, names labels by what they label (`stable_names`), and records
# where each function begins. So it is cut at function boundaries the functions
# themselves choose, each piece with the text's opening and closing directives, plus
# one piece for the rest (literals, tables, globals). A piece whose text an earlier
# build assembled takes its object from the build cache; the others are assembled on
# separate threads. The pieces' objects are what the program links, in order, the
# same whichever were reused, and the list of their keys is kept under the build's
# key (`-n.m`), so a build whose sources did not change links them again without
# generating anything.
#
#==============================================================================================
import back.assemble.assembler
import back.parallel
import c
import luce_std.files
import luce_std.process
import io
import memory
import support.buffer
import support.cache
import support.host_tools
import support.list
import strings
from support.arena import Arena
# mark: Cutting and assembling ==============================================================
## Where pieces are cut. Past `least` bytes of functions, a function whose tag
## (`stable_names`) is a multiple of `cut_every` starts the next piece, and one starts at
## least every four times `least`. A cut depends on the functions around it, not on its
## place in the program, so an edit moves the cuts near it only, and the other pieces keep
## their text, and their objects in the cache. `least` is a sixty-fourth of the functions'
## text, as a power of two between `smallest` and `largest`: an edit reassembles a small
## piece of a small program, while a large one is not linked from thousands of objects
## (each also costs the assembler's start, a few milliseconds). It changes only when the
## program's size crosses a power of two.
let cut_every: u64 = 8
let smallest: usize = 16384
let largest: usize = 1048576
## The least bytes of functions in a piece of a program whose functions' text is `total`.
func least_for(total: usize) -> usize:
var least = smallest
while least < largest and least * 2 * 64 <= total:
least *= 2
return least
## The objects of the pieces the build `key` was assembled in, in order, taken into the
## workspace `dir` (`cache.take`) when the cache still has every one; none otherwise.
pub func cached(key: str, dir: str) -> str[]?!:
let text = (str)(try cache.read(key, "-n.m") else return none)
var objects = try list.List[str].create(64)
var at: usize = 0
while at < text.length:
var end = at
while end < text.length and text.bytes[end] != '\n':
end += 1
let piece = (str)text.bytes[at..<end]
at = end + 1
if piece.length == 0:
continue
let object = try cache.take(piece, "-p.o", dir, true) else return none
try objects.push(object)
if objects.count == 0:
return none
return objects.span()
## Assemble `text` (generated with `split`; `starts`, `tags` and `ending` from its
## generator) in `dir`, and answer the objects to link, in order: a piece whose text is in
## the build cache (`support.cache`) reuses its object, the others are assembled in
## parallel and kept there, and the list of pieces is kept under the build's `key`.
## `macho` assembles through Apple's driver for `arch`; `own` through the compiler's own
## assembler (`back.assemble`), in this process, from text that is never written. A `whole`
## text is one piece: its module-level assembly may define labels local to the file that
## its functions name.
pub func assemble(dir: str, text: str, starts: const usize[], tags: const u64[], ending: usize, key: str, macho: bool, arch: str, whole: bool,
own: bool) -> str[]!:
let functions = starts.length - 1
let rest = starts[0] if whole else starts[functions]
let prelude = (str)text.bytes[..<starts[0]]
let closing = (str)text.bytes[ending..]
# piece k holds functions [cuts[k], cuts[k + 1]); the last piece is the rest
var cuts = try list.List[usize].create(64)
if not whole:
try cuts.push(0)
var since = starts[0]
let least = least_for(rest - starts[0])
for index in 1..<functions:
let size = starts[index] - since
if (size >= least and tags[index] % cut_every == 0) or size >= 4 * least:
try cuts.push(index)
since = starts[index]
try cuts.push(functions)
let made = cuts.count - 1 if cuts.count > 0 else 0
let objects = try new str[made + 1] ---
let keys = try new str[made + 1] ---
let middles = try new str[made + 1] ---
let closings = try new str[made + 1] ---
var pending = try list.List[usize].create(made + 1)
var listing = try buffer.Buffer.create(20 * (made + 1))
for k in 0..<made + 1:
var middle = (str)text.bytes[rest..]
var ends = ""
if k < made:
let to_byte = starts[cuts.get(k + 1)] if cuts.get(k + 1) < functions else rest
middle = (str)text.bytes[starts[cuts.get(k)]..<to_byte]
ends = closing
keys[k] = try cache.piece_key(prelude, middle, ends, arch)
middles[k] = middle
closings[k] = ends
try listing.put(keys[k])
try listing.put("\n")
if let kept = try cache.take(keys[k], "-p.o", dir, true):
objects[k] = kept
continue
# a piece is named by its key, as the cache names it, and relative to the workspace
# the assembler runs in: an ELF object records the file it was assembled from, so
# the program is the same whichever pieces were assembled and which came from the
# cache
objects[k] = try strings.format(try new u8[dir.length + keys[k].length + 8] ---, f"{dir}/{keys[k]}-p.o")
if not own:
try write_piece(try strings.format(try new u8[dir.length + keys[k].length + 8] ---, f"{dir}/{keys[k]}-p.s"), prelude, middle, ends)
try pending.push(k)
let texts = Texts(prelude = prelude, middles = middles, closings = closings, objects = objects)
try assemble_pending(dir, keys, pending.span(), macho, &texts if own else none)
for k in pending.span():
cache.keep(keys[k], "-p.o", objects[k])
cache.keep_bytes_as(key, "-n.m", listing.bytes())
if cache.report and try cache.on():
try io.stderr().write(f"luce-base: {made + 1 - pending.count} of {made + 1} pieces from the cache\n")
return objects
## `prelude`, `middle` and `closing` as one file at `path`.
func write_piece(path: str, prelude: str, middle: str, closing: str) -> !:
with memory.heap:
var whole = try joined(prelude, middle, closing)
defer free(whole)
try files.write((c.str)path, whole)
## `prelude`, `middle` and `closing` as one text, in the current allocator.
func joined(prelude: str, middle: str, closing: str) -> u8[]!:
var whole = try new u8[prelude.length + middle.length + closing.length] ---
memory.copy(whole, prelude.bytes, prelude.length)
memory.copy(whole[prelude.length..], middle.bytes, middle.length)
memory.copy(whole[prelude.length + middle.length..], closing.bytes, closing.length)
return whole
## The pieces' texts and objects, by piece, for the compiler's own assembler.
struct Texts:
var prelude: str
var middles: const str[]
var closings: const str[]
var objects: const str[]
# mark: The assembler threads ===============================================================
## What the threads assembling the pieces share: each batch is one run of the assembler.
struct Assembling:
var dir: str
var keys: const str[]
## The pieces to assemble, by index into `keys`, and where each batch's run of them
## begins in it (one more than the batches, the last its end).
var pieces: const usize[]
var batches: usize[]
## Apple's driver, which takes many sources in one run; else `as`, one each.
var driver: str
## The texts the compiler's own assembler takes, when it assembles them.
var texts: const Texts*?
var failures: parallel.Failure[]
## Assemble the pieces `pending` lists in `dir`, as `<key>-p.o` from `<key>-p.s`: on macOS
## in one run of the driver per thread (each run starts in some milliseconds; an edit's
## few pieces take one each), else one `as` per piece; or from `texts`, in this process,
## a batch per thread.
func assemble_pending(dir: str, keys: const str[], pending: const usize[], macho: bool, texts: const Texts*?) -> !:
if pending.length == 0:
return
let threads = (usize)parallel.workers()
let batched = macho or texts != none
let runs = pending.length if not batched else (threads if threads < pending.length else pending.length)
var batches = try new usize[runs + 1] ---
for r in 0..<runs + 1:
batches[r] = r * pending.length // runs
var job = Assembling(dir = dir, keys = keys, pieces = pending, batches = batches, driver = try driver_for(macho), texts = texts,
failures = try parallel.failures(runs))
with memory.heap:
try parallel.each(runs, assemble_task, (void*)&job)
try parallel.check(job.failures)
func assemble_task(context: void*?, index: usize):
let job = (Assembling*)(context else return)
with memory.heap:
assemble_batch(job, index) catch failure:
job.failures[index] = parallel.failed(failure.code, failure.message)
func assemble_batch(job: Assembling*, index: usize) -> !:
# the names live as long as the run, in an arena of the batch's own
var arena = Arena(memory.heap)
defer arena.destroy()
with arena:
let first = job.batches[index]
let after = job.batches[index + 1]
if let texts = job.texts:
for at in first..<after:
let k = job.pieces[at]
let text = try joined(texts.prelude, texts.middles[k], texts.closings[k])
var object = try buffer.Buffer.create(text.length // 4)
try assembler.assemble(text, job.keys[k], &object)
try files.write((c.str)texts.objects[k], object.bytes())
return
var arguments = try new c.str[3 + 3 * (after - first)] ---
var n: usize = 0
if job.driver.length > 0:
for word in ["-x", "assembler", "-c"]:
arguments[n] = (c.str)word
n += 1
for at in first..<after:
let key = job.keys[job.pieces[at]]
if job.driver.length == 0:
# `as` takes one source and names its object
arguments[n] = "-o"
arguments[n + 1] = (c.str)try strings.format(try new u8[key.length + 8] ---, f"{key}-p.o")
n += 2
# the driver names each object as its source, `<key>-p.o`
arguments[n] = (c.str)try strings.format(try new u8[key.length + 8] ---, f"{key}-p.s")
n += 1
var program: c.str = "as"
if job.driver.length > 0:
program = (c.str)job.driver
let assembled = try host_tools.run(program, arguments[..<n], job.dir)
defer assembled.release()
if assembled.value.exit_code != 0:
try io.stderr().write(assembled.value.error_output)
error(rejected, "the assembler rejected the generated code")
## The driver that assembles the pieces: Apple's when `macho`, else none (`as`, one each).
func driver_for(macho: bool) -> str!:
if macho:
return try apple_driver()
return ""
## Apple's C driver, which `as` runs through a shell script of the toolchain's: found once
## (`xcrun -f clang`), so each run of the assembler starts no `xcrun` and no shell. Its
## `-x assembler -c` is all `as` does.
func apple_driver() -> str!:
if found_driver.length > 0:
return found_driver
var arguments: c.str[2] = ["-f", "clang"]
let found = try process.run("xcrun", arguments)
defer found.release()
if found.value.exit_code != 0:
error(rejected, "xcrun could not find the assembler's driver")
var path = found.value.output
while path.length > 0 and (path.bytes[path.length - 1] == '\n' or path.bytes[path.length - 1] == '\r'):
path = (str)path.bytes[..<path.length - 1]
with memory.heap:
found_driver = try strings.copy(path)
return found_driver
var found_driver: str = ""
## The assembler rejected a piece.
pub let rejected: ErrorCode = ErrorCode.package(43)