Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-canvas

Source of release 0.5.1

0.5.1 / src / tiles / scratch.lucb

#==============================================================================================
#
#   tiles/scratch - The disk tier: tiles past the RAM budget, in a scratch file
#
#   DESCRIPTION:
#       When the RAM copies of tiles outgrow the RAM budget, the governor moves the least
#       wanted of those not on the GPU (history's first) to one scratch file: each tile is a
#       slot of `tile_bytes` at a slot-aligned offset, written once (a tile never changes)
#       and read back when the tile is wanted again, which leaves the slot in place so moving
#       it out once more costs nothing. A slot freed with its tile goes on a free list and is
#       used again before the file grows. The file is made the first time it is needed, in a
#       fresh directory under the scratch directory (`set_scratch_directory`, else a disk-backed
#       default: on Linux the user's cache, since /tmp is often RAM-backed tmpfs there, which
#       would put the spilled tiles back in memory; elsewhere the system's temporary one). It
#       is opened as a temporary file, gone with its last handle whatever way the process
#       ends: on macOS and Linux its name is removed at once; on Windows the system deletes
#       it when the handle closes. A Windows folder of it can outlive the process, so a new
#       scratch file first sweeps stale ones away; `release_scratch` removes this one on a
#       clean exit.
#
#==============================================================================================

## The scratch file, once made; its next unused offset; slots given back.
var scratch: files.File*? = none
var scratch_end: i64 = 0
var free_slots: i64[]
var free_count: usize = 0
## Bytes written into slots (a compressed tile's own length), and tiles
## written out and read back.
var disk_tile_bytes: usize = 0
var residency_spills: u64 = 0
var residency_loads: u64 = 0
## Tiles brought back from a document file (tiles/backing).
var residency_fetches: u64 = 0

## The scratch directory the application chose; empty for the default.
var chosen_root: str = ""

## Keep scratch files in `directory` (which must exist); "" for the default. It
## applies to a scratch file not yet made: one already open stays where it is.
pub func set_scratch_directory(directory: str) -> !:
    let copied = try strings.copy(directory)
    if chosen_root.length > 0:
        strings.release(chosen_root)
    chosen_root = copied

## The directory scratch files go in: the chosen one, else a disk-backed default.
func scratch_root() -> interop.Owned[str]!:
    if chosen_root.length > 0:
        return interop.Owned[str](chosen_root)
    if platform.linux:
        return try linux_cache_root()
    else:
        return interop.Owned[str](temporary_root())

## Linux: $XDG_CACHE_HOME/luce, else ~/.cache/luce, made if missing; /tmp when
## there is no home at all.
func linux_cache_root() -> interop.Owned[str]!:
    var root = interop.Owned[str](temporary_root())
    if let chosen = os.env((c.str)"XDG_CACHE_HOME"):
        root = try paths.join(chosen, "luce")
    elif let home = os.env((c.str)"HOME"):
        root = try paths.join(home, ".cache/luce")
    else:
        return root
    errdefer root.release()
    try files.ensure_directory(root.value)
    return root

## The directory temporary files go in: TMPDIR or TEMP, else /tmp.
func temporary_root() -> str:
    if let chosen = os.env((c.str)"TMPDIR"):
        return chosen
    if let chosen = os.env((c.str)"TEMP"):
        return chosen
    return "/tmp"

## The name of the scratch file inside its folder.
let scratch_name: str = "tiles.scratch"

## The folder holding this process's scratch file while one is open (Windows).
var scratch_folder: interop.Owned[str]? = none

## The scratch file, made on first use.
func scratch_file() -> files.File*!:
    if let open = scratch:
        return open
    let root = try scratch_root()
    defer root.release()
    sweep_stale(root.value)
    let directory = try files.create_temporary_directory((c.str)root.value)
    errdefer directory.release()
    let path = try paths.join(directory.value, scratch_name)
    defer path.release()
    var opened = try files.File.open((c.str)path.value, files.OpenMode.create_temporary, permissions = 384)
    errdefer opened.destroy()
    let kept = try new files.File in memory.heap
    *kept = opened
    scratch = kept
    keep_folder(directory)
    return kept

## macOS and Linux: the file has no name already, so its folder goes now.
## Windows: the file keeps its name until it closes, so its folder is kept
## for `release_scratch`.
func keep_folder(directory: interop.Owned[str]):
    if platform.windows:
        scratch_folder = directory
    else:
        files.remove_directory((c.str)directory.value) catch failure:
            _ = failure
        directory.release()

## On a clean exit: close the scratch file (Windows deletes it then) and remove
## its folder. Tiles moved to it are gone, so the documents must be closed first.
pub func release_scratch():
    if let open = scratch:
        open.destroy()
        free(open) in memory.heap
        scratch = none
    scratch_end = 0
    free_count = 0
    disk_tile_bytes = 0
    if let folder = scratch_folder:
        files.remove_directory((c.str)folder.value) catch failure:
            _ = failure
        folder.release()
        scratch_folder = none

## Windows: remove the folders earlier runs left under `root` (a killed process
## leaves its folder; the system already deleted the file inside unless the run
## predates temporary files). A folder is taken only when it is one of ours by
## name and holds nothing but a scratch file; a file another running instance
## still has open cannot go, and its folder stays.
func sweep_stale(root: str):
    if platform.windows:
        let names = files.list((c.str)root) catch failure:
            _ = failure
            return
        defer names.release()
        for name in names.value:
            if not is_scratch_folder(name):
                continue
            let folder = paths.join(root, name) catch failure:
                _ = failure
                continue
            remove_stale_folder(folder.value)
            folder.release()
    else:
        return

## Remove `folder` if it holds at most a scratch file.
func remove_stale_folder(folder: str):
    let inside = files.list((c.str)folder) catch failure:
        _ = failure
        return
    defer inside.release()
    if not only_scratch(inside.value):
        return
    if inside.value.length == 1:
        let file = paths.join(folder, scratch_name) catch failure:
            _ = failure
            return
        defer file.release()
        files.remove_file((c.str)file.value) catch failure:
            _ = failure
            return
    files.remove_directory((c.str)folder) catch failure:
        _ = failure

## Whether `name` is a folder `create_temporary_directory` makes: `.luce-` and 32
## lower-case hexadecimal digits.
func is_scratch_folder(name: str) -> bool:
    if name.length != 38 or not strings.starts_with(name, ".luce-"):
        return false
    for index in 6..<name.length:
        let byte = name.bytes[index]
        if not ((byte >= '0' and byte <= '9') or (byte >= 'a' and byte <= 'f')):
            return false
    return true

## Whether a folder's entries are nothing but (at most) the scratch file.
func only_scratch(names: const str[]) -> bool:
    return names.length == 0 or (names.length == 1 and names[0] == scratch_name)

## A slot for one tile: a freed one, else the end of the file.
func take_slot() -> i64!:
    if free_count > 0:
        free_count -= 1
        return free_slots[free_count]
    let offset = scratch_end
    scratch_end += (i64)tile_bytes
    return offset

func give_slot(offset: i64) -> !:
    if free_count == free_slots.length:
        let grown = try new i64[free_slots.length * 2 + 64] --- in memory.heap
        memory.copy(grown, free_slots, free_count)
        if free_slots.length > 0:
            free(free_slots) in memory.heap
        free_slots = grown
    free_slots[free_count] = offset
    free_count += 1

## Move `tile`'s RAM copy to the scratch file (writing it only the first time)
## and free the RAM. A tile on the GPU keeps its texture.
func spill(tile: Tile*) -> !:
    if tile.ram.length == 0:
        return
    # The document file holds it: nothing to write.
    if tile.disk < 0 and not backed(tile):
        let file = try scratch_file()
        let offset = try take_slot()
        errdefer try_give_slot(offset)
        _ = try file.seek(offset)
        var written: usize = 0
        while written < tile_bytes:
            let count = try file.write(tile.ram[written..])
            if count == 0:
                error(invalid, "the scratch file took no more bytes")
            written += count
        tile.disk = offset
        disk_tile_bytes += tile_bytes
    drop_ram_copy(tile)
    residency_spills += 1

## Move `tile`'s compressed copy to the scratch file: only its compressed
## bytes are written, into a slot of a whole tile's size (the free list stays
## one kind of slot); the RAM it held is freed. The document file holding it
## means nothing need be written.
func spill_packed(tile: Tile*) -> !:
    if tile.packed.length == 0:
        return
    if tile.disk < 0 and not backed(tile):
        let file = try scratch_file()
        let offset = try take_slot()
        errdefer try_give_slot(offset)
        _ = try file.seek(offset)
        var written: usize = 0
        while written < tile.packed.length:
            let count = try file.write(tile.packed[written..])
            if count == 0:
                error(invalid, "the scratch file took no more bytes")
            written += count
        tile.disk = offset
        tile.disk_length = (u32)tile.packed.length
        disk_tile_bytes += tile.packed.length
    drop_packed(tile)
    residency_spills += 1

## Bring a tile whose slot holds it compressed back into RAM, decoded; the slot stays.
func load_packed_slot(tile: Tile*) -> !:
    let decode = codec_decode else error(invalid, "a compressed tile came back without a codec")
    let file = try scratch_file()
    let packed = try new u8[(usize)tile.disk_length] --- in memory.heap
    defer free(packed) in memory.heap
    _ = try file.seek(tile.disk)
    var read: usize = 0
    while read < packed.length:
        let count = try file.read(packed[read..])
        if count == 0:
            error(invalid, "the scratch file ended inside a tile")
        read += count
    let copy = try new u8[tile_bytes] --- in memory.heap
    errdefer free(copy) in memory.heap
    memory.set(copy, 0)
    try decode(codec_context, packed, tile_size, copy)
    tile.ram = copy
    ram_tile_bytes += tile_bytes
    residency_loads += 1

func try_give_slot(offset: i64):
    give_slot(offset) catch failure:
        _ = failure

## Bring `tile`'s bytes back into RAM from its slot, if they are only on disk;
## the slot stays, so spilling it again is free.
func ensure_ram(tile: Tile*) -> !:
    if tile.ram.length > 0:
        return
    if tile.packed.length > 0:
        try unpack(tile)
        return
    if tile.disk < 0:
        try fetch_backed(tile)
        return
    if tile.disk_length > 0:
        try load_packed_slot(tile)
        return
    let file = try scratch_file()
    let copy = try new u8[tile_bytes] --- in memory.heap
    errdefer free(copy) in memory.heap
    _ = try file.seek(tile.disk)
    var read: usize = 0
    while read < tile_bytes:
        let count = try file.read(copy[read..])
        if count == 0:
            error(invalid, "the scratch file ended inside a tile")
        read += count
    tile.ram = copy
    ram_tile_bytes += tile_bytes
    residency_loads += 1

## Free `tile`'s slot: the tile is gone.
func release_disk(tile: Tile*):
    if tile.disk < 0:
        return
    let slot = tile.disk
    tile.disk = -1
    let written = (usize)tile.disk_length if tile.disk_length > 0 else tile_bytes
    tile.disk_length = 0
    # After `release_scratch` the file and its accounting are gone; tiles an
    # undo history or a closing document still holds are freed later.
    if scratch == none:
        return
    try_give_slot(slot)
    disk_tile_bytes -= written if disk_tile_bytes >= written else disk_tile_bytes