Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-prism

Source of release 0.6.1

0.6.1 / src / wal / journal.lucb

## Append-only v1 journal. Exactly one process owns the persistent sidecar lock.
## Partial final frames are discarded on reopen; complete corrupt frames fail.
import c
from luce_std import files
import io
import memory
import platform
from luce_std import paths
import strings
import wal.tree
import wal.storage_io
import wal.checkpoint
from wal.bytes import Buffer, Reader, checksum, compare, valid_key, max_key, max_value, max_frame, corrupt, invalid, busy, limit, uncertain

pub let header: str = "PRISM-WAL-001\n"
pub let max_journal: i64 = 268435456

pub struct Journal:
    pub var file: files.File
    var lock_file: files.File
    var directory: files.Directory
    pub var position: i64
    pub var recovered_tail: i64
    var controls: storage_io.Controls
    var filename: str
    var name_owner: memory.Allocator?

    pub func close():
        self.file.destroy()
        self.directory.destroy()
        self.lock_file.destroy()
        if self.filename.length > 0:
            let owner = self.name_owner else trap("journal name lost its allocator")
            with owner: strings.release(self.filename)
            self.filename = ""
            self.name_owner = none

    pub func append(frame: const u8[]) -> !:
        if (i64)frame.length > max_journal - self.position:
            error(limit, "journal reached 256 MiB; export to a fresh database")
        try storage_io.write_all(&self.file, self.controls, frame)
        try self.controls.sync(self.controls.context, &self.file)
        self.position += (i64)frame.length

    ## Serialize with append at the engine layer; readers own independent roots.
    pub func compact(root: tree.Node*?, generation: u64) -> !:
        let size = try checkpoint.measure(root, generation)
        try self.check_identity()
        var candidate: u8[64]
        var name: str
        var replacement: files.File
        defer replacement.destroy()
        var owned = false
        defer cleanup_if_owned(&self.directory, self.controls, &replacement, &name, &owned)
        for attempt in 0..<128:
            var random: u8[16]
            try self.controls.random(self.controls.context, random)
            var encoded_name: u8[32]
            let digits = "0123456789abcdef".bytes
            for i in 0..<16:
                encoded_name[i * 2] = digits[random[i] >> 4]
                encoded_name[i * 2 + 1] = digits[random[i] & 15]
            name = try strings.format(candidate, f".luce-db-{(str)encoded_name}")
            replacement = self.controls.create(self.controls.context, &self.directory, (c.str)name) catch failure:
                if failure.code == files.already_exists: continue
                error(failure.code, failure.message)
            owned = true
            break
        if not owned: error(busy, "no unused checkpoint temporary name")
        let metadata = try replacement.metadata()
        if metadata.kind != files.FileKind.regular or metadata.links != 1:
            error(invalid, "checkpoint temporary file is not regular/single-link")
        try checkpoint.write(&replacement, self.controls, root, generation, size)
        try self.controls.sync(self.controls.context, &replacement)
        if (try replacement.metadata()).size != size: error(invalid, "checkpoint file size mismatch")
        try self.check_identity()
        # ReplaceFileW deliberately opens the replacement with no sharing mode.
        # Close our synced handle before publication, then reopen the published
        # identity below. Unix rename keeps the open descriptor throughout.
        if platform.windows:
            try self.controls.close(self.controls.context, &replacement)
        var published = false
        self.controls.replace(self.controls.context, &self.directory, (c.str)name, (c.str)self.filename, &published) catch failure:
            if published: error(uncertain, "checkpoint replacement outcome uncertain; reopen")
            error(failure.code, failure.message)
        owned = false
        if platform.windows:
            replacement = self.directory.open_file((c.str)self.filename, files.OpenMode.read_write,
                follow_symlinks = false, nonblocking = true) catch failure:
                error(uncertain, "checkpoint published but could not be reopened; reopen")
            let positioned = replacement.seek(size) catch failure:
                error(uncertain, "checkpoint published but append position could not be restored; reopen")
            _ = positioned
        ## Success means publication regardless of whether a custom hook set its flag.
        let published_metadata = self.directory.entry_metadata((c.str)self.filename, follow_symlinks = false) catch failure:
            error(uncertain, "checkpoint published pathname cannot be verified; reopen")
        if published_metadata.kind != files.FileKind.regular or published_metadata.device != metadata.device or published_metadata.inode != metadata.inode or published_metadata.links != 1:
            error(uncertain, "checkpoint published identity changed; reopen")
        self.controls.directory_sync(self.controls.context, &self.directory) catch failure:
            error(uncertain, "checkpoint directory sync failed after publication; reopen")
        var old = self.file
        defer old.destroy()
        self.file = replacement
        replacement = files.File()
        self.controls.close(self.controls.context, &old) catch failure:
            error(uncertain, "checkpoint old journal close failed after publication; reopen")
        self.position = size
        self.recovered_tail = 0

    func check_identity() -> !:
        let opened = self.file.metadata() catch failure:
            error(uncertain, "journal descriptor identity unavailable; reopen")
        let named = self.directory.entry_metadata((c.str)self.filename, follow_symlinks = false) catch failure:
            error(uncertain, "journal pathname identity unavailable; reopen")
        if opened.kind != files.FileKind.regular or opened.links != 1 or opened.size != self.position or named.kind != files.FileKind.regular or named.links != 1 or opened.device != named.device or opened.inode != named.inode:
            error(uncertain, "journal pathname/identity changed; checkpoint refused, reopen")

func cleanup_owned(directory: files.Directory*, controls: storage_io.Controls, file: files.File*, name: str) -> !:
    let opened = try file.metadata()
    let named = directory.entry_metadata((c.str)name, follow_symlinks = false) catch failure:
        if failure.code == files.missing: return
        error(failure.code, failure.message)
    if named.kind != files.FileKind.regular or named.device != opened.device or named.inode != opened.inode:
        error(invalid, "checkpoint temporary name changed; cleanup refused")
    try controls.remove(controls.context, directory, (c.str)name)

func cleanup_if_owned(directory: files.Directory*, controls: storage_io.Controls, file: files.File*, name: str*, owned: bool*):
    if *owned:
        cleanup_owned(directory, controls, file, *name) catch failure:
            _ = failure ## Original error wins; an owned orphan may remain.

pub func open(path: str, controls: storage_io.Controls = storage_io.Controls()) -> Journal!:
    if path.length == 0 or strings.contains(path, "\0"):
        error(invalid, "invalid database path")
    let directory_name = try strings.copy(paths.directory(path))
    defer strings.release(directory_name)
    let name_owner = memory.heap
    var file_name: str
    with name_owner: file_name = try strings.copy(paths.base(path))
    errdefer release_name(file_name, name_owner)
    if file_name == "" or file_name == "." or file_name == "..":
        error(invalid, "database path needs a filename")
    var scratch: u8[4096]
    let lock_name = try strings.format(scratch, f"{file_name}.lock")
    var directory = try files.Directory.open((c.str)directory_name, follow_symlinks = false)
    errdefer directory.destroy()
    var lock_file = try storage_io.open_lock_file(&directory, (c.str)lock_name)
    errdefer lock_file.destroy()
    if (try lock_file.metadata()).kind != files.FileKind.regular:
        error(invalid, "database lock is not a regular file")
    try storage_io.lock_file(&lock_file)
    var created = false
    var file = directory.open_file((c.str)file_name, files.OpenMode.read_write, follow_symlinks = false, nonblocking = true) catch failure:
        if failure.code != files.missing: error(failure.code, failure.message)
        created = true
        recover try directory.open_file((c.str)file_name, files.OpenMode.create_new_read_write, 384, follow_symlinks = false, nonblocking = true)
    errdefer file.destroy()
    let metadata = try file.metadata()
    if metadata.kind != files.FileKind.regular or metadata.links != 1:
        error(invalid, "database must be a regular file with one link")
    if created:
        try storage_io.write_all(&file, controls, header.bytes)
    _ = try file.seek(0)
    return Journal(file = file, lock_file = lock_file, directory = directory, controls = controls, filename = file_name, name_owner = name_owner)

func release_name(name: str, owner: memory.Allocator):
    with owner: strings.release(name)

pub func encode_frame(payload: const u8[], generation: u64, operations: usize) -> Buffer!:
    var out: Buffer
    errdefer out.close()
    try out.put(b"TXN1")
    try out.uint((u64)payload.length, 4)
    try out.uint(generation, 8)
    try out.uint((u64)operations, 4)
    try out.uint((u64)checksum(out.span()), 4)
    try out.put(payload)
    let crc = checksum(out.span())
    try out.uint((u64)crc, 4)
    try out.put(b"END1")
    return out

pub func operation(out: Buffer*, key: str, value: const u8[], deleted: bool) -> !:
    try out.reserve(out.count + 9 + key.length + value.length)
    try out.uint(2 if deleted else 1, 1)
    try out.uint((u64)key.length, 4)
    try out.uint((u64)value.length, 4)
    try out.put(key.bytes)
    try out.put(value)

func apply(payload: const u8[], operations: usize, root: tree.Node*?) -> tree.Node*?!:
    var next = tree.hold(root)
    errdefer tree.drop(next)
    var reader = Reader(bytes = payload)
    if operations == 0 or operations > 65536:
        error(corrupt, "invalid database operation count")
    for index in 0..<operations:
        let kind = try reader.uint(1)
        let key_length = (usize)try reader.uint(4)
        let value_length = (usize)try reader.uint(4)
        if key_length == 0 or key_length > max_key or value_length > max_value or (kind != 1 and kind != 2) or (kind == 2 and value_length != 0):
            error(corrupt, "invalid database operation")
        let key = try reader.take(key_length)
        valid_key((str)key) catch failure: error(corrupt, "invalid persisted key")
        let value = try reader.take(value_length)
        var replacement: tree.Node*?
        if kind == 1:
            let item = try tree.entry(key, value)
            defer tree.drop_entry(item)
            replacement = try tree.put(next, item)
        else:
            replacement = try tree.remove(next, key)
        tree.drop(next)
        next = replacement
    if reader.offset != payload.length:
        error(corrupt, "extra bytes in database transaction")
    return next

pub func replay(log: Journal*, root: tree.Node*?*, generation: u64*) -> !:
    let size = (try log.file.metadata()).size
    if size < (i64)header.length or size > max_journal:
        error(corrupt, "invalid database journal size")
    var magic: u8[14]
    try storage_io.read_exact(&log.file, log.controls, magic)
    log.position = 14
    if compare(magic, checkpoint.header.bytes) == 0:
        let base = try checkpoint.read(&log.file, log.controls, size)
        tree.drop(*root)
        *root = base.root
        *generation = base.generation
        log.position = base.end
    elif compare(magic, header.bytes) != 0:
        error(corrupt, "unknown database format")
    while log.position < size:
        let remaining = size - log.position
        if remaining < 24: break
        var prefix: u8[24]
        try storage_io.read_exact(&log.file, log.controls, prefix)
        var reader = Reader(bytes = prefix)
        if compare(try reader.take(4), b"TXN1") != 0: error(corrupt, "invalid transaction header")
        let length = (usize)try reader.uint(4)
        let sequence = try reader.uint(8)
        let operations = (usize)try reader.uint(4)
        let prefix_crc = try reader.uint(4)
        if prefix_crc != (u64)checksum(prefix[..<20]) or sequence != *generation + 1 or sequence > 9223372036854775807:
            error(corrupt, "invalid transaction sequence/checksum")
        if length > max_frame - 32: error(corrupt, "invalid transaction size")
        if (i64)length + 32 > remaining: break
        var bytes: Buffer
        defer bytes.close()
        try bytes.reserve(length + 32)
        try bytes.put(prefix)
        try storage_io.read_exact(&log.file, log.controls, bytes.data[24..<length + 32])
        bytes.count = length + 32
        var trailer = Reader(bytes = bytes.span()[24 + length..])
        if try trailer.uint(4) != (u64)checksum(bytes.span()[..<24 + length]) or compare(try trailer.take(4), b"END1") != 0:
            error(corrupt, "invalid transaction trailer/checksum")
        let next = try apply(bytes.span()[24..<24 + length], operations, *root)
        tree.drop(*root)
        *root = next
        *generation = sequence
        log.position += (i64)length + 32
    if log.position < size:
        log.recovered_tail = size - log.position
        try log.controls.truncate(log.controls.context, &log.file, log.position)
    ## Reopen can see complete bytes from an unacknowledged write or creation
    ## whose previous sync failed. Re-establish BOTH durability boundaries before
    ## publishing recovered state, even when the existing journal has no torn tail.
    try log.controls.sync(log.controls.context, &log.file)
    try log.controls.directory_sync(log.controls.context, &log.directory)
    _ = try log.file.seek(log.position)