dymokomi/luce-prism
Source of release 0.6.0
0.6.0 / src / wal / journal.lucb
## Append-only v1 journal. Exactly one process owns the persistent sidecar lock.
## Partial final frames are discarded on reopen; complete corrupt frames fail.
import c
from luce_std import files
import io
import memory
import platform
from luce_std import paths
import strings
import wal.tree
import wal.storage_io
import wal.checkpoint
from wal.bytes import Buffer, Reader, checksum, compare, valid_key, max_key, max_value, max_frame, corrupt, invalid, busy, limit, uncertain
pub let header: str = "PRISM-WAL-001\n"
pub let max_journal: i64 = 268435456
pub struct Journal:
pub var file: files.File
var lock_file: files.File
var directory: files.Directory
pub var position: i64
pub var recovered_tail: i64
var controls: storage_io.Controls
var filename: str
var name_owner: memory.Allocator?
pub func close():
self.file.destroy()
self.directory.destroy()
self.lock_file.destroy()
if self.filename.length > 0:
let owner = self.name_owner else trap("journal name lost its allocator")
with owner: strings.release(self.filename)
self.filename = ""
self.name_owner = none
pub func append(frame: const u8[]) -> !:
if (i64)frame.length > max_journal - self.position:
error(limit, "journal reached 256 MiB; export to a fresh database")
try storage_io.write_all(&self.file, self.controls, frame)
try self.controls.sync(self.controls.context, &self.file)
self.position += (i64)frame.length
## Serialize with append at the engine layer; readers own independent roots.
pub func compact(root: tree.Node*?, generation: u64) -> !:
let size = try checkpoint.measure(root, generation)
try self.check_identity()
var candidate: u8[64]
var name: str
var replacement: files.File
defer replacement.destroy()
var owned = false
defer cleanup_if_owned(&self.directory, self.controls, &replacement, &name, &owned)
for attempt in 0..<128:
var random: u8[16]
try self.controls.random(self.controls.context, random)
var encoded_name: u8[32]
let digits = "0123456789abcdef".bytes
for i in 0..<16:
encoded_name[i * 2] = digits[random[i] >> 4]
encoded_name[i * 2 + 1] = digits[random[i] & 15]
name = try strings.format(candidate, f".luce-db-{(str)encoded_name}")
replacement = self.controls.create(self.controls.context, &self.directory, (c.str)name) catch failure:
if failure.code == files.already_exists: continue
error(failure.code, failure.message)
owned = true
break
if not owned: error(busy, "no unused checkpoint temporary name")
let metadata = try replacement.metadata()
if metadata.kind != files.FileKind.regular or metadata.links != 1:
error(invalid, "checkpoint temporary file is not regular/single-link")
try checkpoint.write(&replacement, self.controls, root, generation, size)
try self.controls.sync(self.controls.context, &replacement)
if (try replacement.metadata()).size != size: error(invalid, "checkpoint file size mismatch")
try self.check_identity()
# ReplaceFileW deliberately opens the replacement with no sharing mode.
# Close our synced handle before publication, then reopen the published
# identity below. Unix rename keeps the open descriptor throughout.
if platform.windows:
try self.controls.close(self.controls.context, &replacement)
var published = false
self.controls.replace(self.controls.context, &self.directory, (c.str)name, (c.str)self.filename, &published) catch failure:
if published: error(uncertain, "checkpoint replacement outcome uncertain; reopen")
error(failure.code, failure.message)
owned = false
if platform.windows:
replacement = self.directory.open_file((c.str)self.filename, files.OpenMode.read_write,
follow_symlinks = false, nonblocking = true) catch failure:
error(uncertain, "checkpoint published but could not be reopened; reopen")
let positioned = replacement.seek(size) catch failure:
error(uncertain, "checkpoint published but append position could not be restored; reopen")
_ = positioned
## Success means publication regardless of whether a custom hook set its flag.
let published_metadata = self.directory.entry_metadata((c.str)self.filename, follow_symlinks = false) catch failure:
error(uncertain, "checkpoint published pathname cannot be verified; reopen")
if published_metadata.kind != files.FileKind.regular or published_metadata.device != metadata.device or published_metadata.inode != metadata.inode or published_metadata.links != 1:
error(uncertain, "checkpoint published identity changed; reopen")
self.controls.directory_sync(self.controls.context, &self.directory) catch failure:
error(uncertain, "checkpoint directory sync failed after publication; reopen")
var old = self.file
defer old.destroy()
self.file = replacement
replacement = files.File()
self.controls.close(self.controls.context, &old) catch failure:
error(uncertain, "checkpoint old journal close failed after publication; reopen")
self.position = size
self.recovered_tail = 0
func check_identity() -> !:
let opened = self.file.metadata() catch failure:
error(uncertain, "journal descriptor identity unavailable; reopen")
let named = self.directory.entry_metadata((c.str)self.filename, follow_symlinks = false) catch failure:
error(uncertain, "journal pathname identity unavailable; reopen")
if opened.kind != files.FileKind.regular or opened.links != 1 or opened.size != self.position or named.kind != files.FileKind.regular or named.links != 1 or opened.device != named.device or opened.inode != named.inode:
error(uncertain, "journal pathname/identity changed; checkpoint refused, reopen")
func cleanup_owned(directory: files.Directory*, controls: storage_io.Controls, file: files.File*, name: str) -> !:
let opened = try file.metadata()
let named = directory.entry_metadata((c.str)name, follow_symlinks = false) catch failure:
if failure.code == files.missing: return
error(failure.code, failure.message)
if named.kind != files.FileKind.regular or named.device != opened.device or named.inode != opened.inode:
error(invalid, "checkpoint temporary name changed; cleanup refused")
try controls.remove(controls.context, directory, (c.str)name)
func cleanup_if_owned(directory: files.Directory*, controls: storage_io.Controls, file: files.File*, name: str*, owned: bool*):
if *owned:
cleanup_owned(directory, controls, file, *name) catch failure:
_ = failure ## Original error wins; an owned orphan may remain.
pub func open(path: str, controls: storage_io.Controls = storage_io.Controls()) -> Journal!:
if path.length == 0 or strings.contains(path, "\0"):
error(invalid, "invalid database path")
let directory_name = try strings.copy(paths.directory(path))
defer strings.release(directory_name)
let name_owner = memory.heap
var file_name: str
with name_owner: file_name = try strings.copy(paths.base(path))
errdefer release_name(file_name, name_owner)
if file_name == "" or file_name == "." or file_name == "..":
error(invalid, "database path needs a filename")
var scratch: u8[4096]
let lock_name = try strings.format(scratch, f"{file_name}.lock")
var directory = try files.Directory.open((c.str)directory_name, follow_symlinks = false)
errdefer directory.destroy()
var lock_file = try storage_io.open_lock_file(&directory, (c.str)lock_name)
errdefer lock_file.destroy()
if (try lock_file.metadata()).kind != files.FileKind.regular:
error(invalid, "database lock is not a regular file")
try storage_io.lock_file(&lock_file)
var created = false
var file = directory.open_file((c.str)file_name, files.OpenMode.read_write, follow_symlinks = false, nonblocking = true) catch failure:
if failure.code != files.missing: error(failure.code, failure.message)
created = true
recover try directory.open_file((c.str)file_name, files.OpenMode.create_new_read_write, 384, follow_symlinks = false, nonblocking = true)
errdefer file.destroy()
let metadata = try file.metadata()
if metadata.kind != files.FileKind.regular or metadata.links != 1:
error(invalid, "database must be a regular file with one link")
if created:
try storage_io.write_all(&file, controls, header.bytes)
_ = try file.seek(0)
return Journal(file = file, lock_file = lock_file, directory = directory, controls = controls, filename = file_name, name_owner = name_owner)
func release_name(name: str, owner: memory.Allocator):
with owner: strings.release(name)
pub func encode_frame(payload: const u8[], generation: u64, operations: usize) -> Buffer!:
var out: Buffer
errdefer out.close()
try out.put(b"TXN1")
try out.uint((u64)payload.length, 4)
try out.uint(generation, 8)
try out.uint((u64)operations, 4)
try out.uint((u64)checksum(out.span()), 4)
try out.put(payload)
let crc = checksum(out.span())
try out.uint((u64)crc, 4)
try out.put(b"END1")
return out
pub func operation(out: Buffer*, key: str, value: const u8[], deleted: bool) -> !:
try out.reserve(out.count + 9 + key.length + value.length)
try out.uint(2 if deleted else 1, 1)
try out.uint((u64)key.length, 4)
try out.uint((u64)value.length, 4)
try out.put(key.bytes)
try out.put(value)
func apply(payload: const u8[], operations: usize, root: tree.Node*?) -> tree.Node*?!:
var next = tree.hold(root)
errdefer tree.drop(next)
var reader = Reader(bytes = payload)
if operations == 0 or operations > 65536:
error(corrupt, "invalid database operation count")
for index in 0..<operations:
let kind = try reader.uint(1)
let key_length = (usize)try reader.uint(4)
let value_length = (usize)try reader.uint(4)
if key_length == 0 or key_length > max_key or value_length > max_value or (kind != 1 and kind != 2) or (kind == 2 and value_length != 0):
error(corrupt, "invalid database operation")
let key = try reader.take(key_length)
valid_key((str)key) catch failure: error(corrupt, "invalid persisted key")
let value = try reader.take(value_length)
var replacement: tree.Node*?
if kind == 1:
let item = try tree.entry(key, value)
defer tree.drop_entry(item)
replacement = try tree.put(next, item)
else:
replacement = try tree.remove(next, key)
tree.drop(next)
next = replacement
if reader.offset != payload.length:
error(corrupt, "extra bytes in database transaction")
return next
pub func replay(log: Journal*, root: tree.Node*?*, generation: u64*) -> !:
let size = (try log.file.metadata()).size
if size < (i64)header.length or size > max_journal:
error(corrupt, "invalid database journal size")
var magic: u8[14]
try storage_io.read_exact(&log.file, log.controls, magic)
log.position = 14
if compare(magic, checkpoint.header.bytes) == 0:
let base = try checkpoint.read(&log.file, log.controls, size)
tree.drop(*root)
*root = base.root
*generation = base.generation
log.position = base.end
elif compare(magic, header.bytes) != 0:
error(corrupt, "unknown database format")
while log.position < size:
let remaining = size - log.position
if remaining < 24: break
var prefix: u8[24]
try storage_io.read_exact(&log.file, log.controls, prefix)
var reader = Reader(bytes = prefix)
if compare(try reader.take(4), b"TXN1") != 0: error(corrupt, "invalid transaction header")
let length = (usize)try reader.uint(4)
let sequence = try reader.uint(8)
let operations = (usize)try reader.uint(4)
let prefix_crc = try reader.uint(4)
if prefix_crc != (u64)checksum(prefix[..<20]) or sequence != *generation + 1 or sequence > 9223372036854775807:
error(corrupt, "invalid transaction sequence/checksum")
if length > max_frame - 32: error(corrupt, "invalid transaction size")
if (i64)length + 32 > remaining: break
var bytes: Buffer
defer bytes.close()
try bytes.reserve(length + 32)
try bytes.put(prefix)
try storage_io.read_exact(&log.file, log.controls, bytes.data[24..<length + 32])
bytes.count = length + 32
var trailer = Reader(bytes = bytes.span()[24 + length..])
if try trailer.uint(4) != (u64)checksum(bytes.span()[..<24 + length]) or compare(try trailer.take(4), b"END1") != 0:
error(corrupt, "invalid transaction trailer/checksum")
let next = try apply(bytes.span()[24..<24 + length], operations, *root)
tree.drop(*root)
*root = next
*generation = sequence
log.position += (i64)length + 32
if log.position < size:
log.recovered_tail = size - log.position
try log.controls.truncate(log.controls.context, &log.file, log.position)
## Reopen can see complete bytes from an unacknowledged write or creation
## whose previous sync failed. Re-establish BOTH durability boundaries before
## publishing recovered state, even when the existing journal has no torn tail.
try log.controls.sync(log.controls.context, &log.file)
try log.controls.directory_sync(log.controls.context, &log.directory)
_ = try log.file.seek(log.position)