dymokomi/luce-base
Source of release 0.45.0
0.45.0 / src / back / reuse.lucb
#==============================================================================================
#
# reuse - Functions an earlier build generated, taken again instead of generated
#
# DESCRIPTION:
# An edit to a small program that imports a large package changes a few of its
# functions; the optimiser and the generator ran again over all of them. With labels
# named by what they label (`stable_names`), a function's text is decided by three
# things: the function as the inliner left it, the functions it calls (the second
# inlining pass opens out a small leaf once it is optimised, and whether a callee is
# a leaf depends on whether its own callees are in the program), and the build's
# settings. Each function's key hashes those (`digest`); a function whose key the
# previous build of the program generated takes that build's text, skipping its
# optimisation and generation. Its callees are still optimised when a function that
# is generated calls them, so what that function opens out is what a build from
# nothing opens out.
#
# The texts are kept in one file per program and settings in the build cache, every
# function of the last build under its key, read whole at the start and replaced at
# the end. A key that does not match generates its function; a file that is missing,
# damaged or of another version is no reuse, never a failure.
#
#==============================================================================================
import back.ir.digest
import back.ir.ir
import back.parallel
import c
import io
import luce_std.files
import memory
import support.buffer
import support.cache
import support.hashed
import support.list
import strings
# mark: Keys ==================================================================================
## Each function's key: the build's `settings`, the function's digest, and for each call it
## makes, the callee's name and, when the program has it, its digest and whether each of its
## own calls names a function of the program.
pub func keys(program: ir.Unit*, settings: digest.Digest) -> digest.Digest[]!:
let n = program.functions.count
var job = Keying(program = program, settings = settings, own = try new digest.Digest[n] ---, keys = try new digest.Digest[n] ---, names = try hashed.Table[str].create(n))
for i in 0..<n:
_ = try job.names.add(program.functions.items[i].name, i)
with memory.heap:
try parallel.each(n, own_task, (void*)&job)
try parallel.each(n, key_task, (void*)&job)
job.names.destroy()
return job.keys
## What the threads computing the keys share.
struct Keying:
var program: ir.Unit*
var settings: digest.Digest
var own: digest.Digest[]
var keys: digest.Digest[]
var names: hashed.Table[str]
func own_task(context: void*?, index: usize):
let job = (Keying*)(context else return)
job.own[index] = digest.function(job.program, &job.program.functions.items[index])
func key_task(context: void*?, index: usize):
let job = (Keying*)(context else return)
var d = digest.Digest.start(0x6b6579)
d.digest(job.settings)
d.digest(job.own[index])
for x in job.program.functions.items[index].instrs.span():
if x.op != ir.Op.call or x.a.kind != ir.RefKind.function:
continue
d.text(x.a.name)
if let g = job.names.find(x.a.name):
d.word(1)
d.digest(job.own[g])
for y in job.program.functions.items[g].instrs.span():
if y.op == ir.Op.call:
d.word(1 if y.a.kind != ir.RefKind.function or job.names.find(y.a.name) != none else 0)
else:
d.word(0)
job.keys[index] = d
# mark: The texts of the last build ===========================================================
## The file's first line; then each function: its key (two little-endian words), the
## generator's flags (a byte), its text's length (a word), and its text.
let header: str = "luce-base-functions-v1\n"
let entry_header: usize = 25
## The texts the last build of this program with these settings generated, by key.
pub struct Kept:
var by_key: hashed.Table[u64]
var entries: list.List[Entry]
## The kept texts under `name` in the build cache when `wanted`, in the current
## allocator; none kept when there are none, or the file is not one this version wrote
## whole.
pub static func load(name: str, wanted: bool) -> Kept!:
var kept = Kept(by_key = try hashed.Table[u64].create(64), entries = try list.List[Entry].create(64))
if not wanted:
return kept
let path = try cache.find(name, "-f.pack", true) else return kept
let read = files.read((c.str)path) catch:
return kept
defer read.release()
let copy = try new u8[read.value.length + 1] ---
memory.copy(copy, read.value, read.value.length)
if not try kept.index(copy[..<read.value.length]):
kept.entries.count = 0
kept.by_key.destroy()
kept.by_key = try hashed.Table[u64].create(64)
return kept
## Index the entries of `b`, which they borrow; false when it is not a whole file of
## this version.
func index(b: const u8[]) -> bool!:
if b.length < header.length or (str)b[..<header.length] != header:
return false
var at = header.length
while at < b.length:
if at + entry_header > b.length:
return false
let low = word64(b, at)
let high = word64(b, at + 8)
let flags = b[at + 16]
let length = (usize)word64(b, at + 17)
at += entry_header
if length > b.length - at:
return false
try self.entries.push(Entry(low = low, high = high, flags = flags, text = (str)b[at..<at + length]))
_ = try self.by_key.add(low, self.entries.count - 1)
at += length
return true
## The kept text of `key`, and its flags, when there is one.
pub func find(key: digest.Digest) -> Entry?:
let k = self.by_key.find(key.low) else return none
let e = self.entries.get(k)
if e.high != key.high:
return none
return e
## One kept function.
pub struct Entry:
## Its key's first word (`digest.Digest`).
pub var low: u64
## Its key's second word.
pub var high: u64
## The generator's flags for it (`Generator.function_flags`).
pub var flags: u8
## Its text.
pub var text: str
## Keep each function's text, `text[starts[i]..<starts[i + 1]]`, and its `flags` under its
## key, as the texts of this build under `name`.
pub func store(name: str, function_keys: const digest.Digest[], text: str, starts: const usize[], flags: const u8[]):
store_texts(name, function_keys, text, starts, flags) catch ignored:
_ = ignored
func store_texts(name: str, function_keys: const digest.Digest[], text: str, starts: const usize[], flags: const u8[]) -> !:
let n = function_keys.length
with memory.heap:
var out = try buffer.Buffer.create(header.length + (starts[n] - starts[0]) + entry_header * n)
defer out.destroy()
try out.put(header)
for i in 0..<n:
let piece = (str)text.bytes[starts[i]..<starts[i + 1]]
try put_word64(&out, function_keys[i].low)
try put_word64(&out, function_keys[i].high)
try out.put_byte(flags[i] if i < flags.length else 0)
try put_word64(&out, (u64)piece.length)
try out.put(piece)
cache.keep_bytes_as(name, "-f.pack", out.bytes())
## The little-endian word at `at`.
func word64(b: const u8[], at: usize) -> u64:
var w: u64 = 0
for k in 0..<8:
w = w | ((u64)b[at + (usize)k] << (u64)(8 * k))
return w
func put_word64(out: buffer.Buffer*, w: u64) -> !:
for k in 0..<8:
try out.put_byte((u8)((w >> (u64)(8 * k)) & 255))
## The name the texts of a program are kept under: its settings' digest, its package and
## its entry as the command spelled it, so programs sharing a cache keep apart.
pub func pack_name(settings: digest.Digest, package_name: str, entry: str) -> str!:
var d = digest.Digest.start(0x7061636b)
d.digest(settings)
d.text(package_name)
d.text(entry)
let room = try new u8[32] ---
let hex = "0123456789abcdef"
for k in 0..<16:
room[k] = hex.bytes[(usize)((d.low >> (u64)(60 - 4 * k)) & 15)]
room[16 + k] = hex.bytes[(usize)((d.high >> (u64)(60 - 4 * k)) & 15)]
return (str)room