Luce packages
LuceLuce BaseLuciaOS

dymokomi/luced-3d

Source of release 0.11.9

0.11.9 / src / script_hosts.luc

## The Script nodes' builds and processes, kept for the app's life by the
## SourceCache the compute worker (and a local network) cooks with. A build
## runs in the background under its key; a cook that is superseded while it
## waits leaves it running, and the next cook of that key waits for the same
## build. Each key keeps one child started and waiting, so a cook pays no
## start-up: the child serves one cook and leaves (each cook starts fresh, as
## Houdini's Python SOP does), and the next is started at once. An input whose
## content key (its node's stamp) is the one the last cook sent keeps its file.
## The disk cache is kept to its limit (script_cache.luc).
from luce_std import files, paths
from luce_geocore.geocore import GeometryFile
from geometry_data import GeometryData
import script_cache
from luce_std.process import Command
from luce_geocore.script_host import ScriptProcess
from script_build import BuildPlan, toolchain, build_failure, scripts_home
from script_packages import editor_packages
import script_build
from luce_ui import ui
from signal import Signal
import time

## A build in flight or done.
pub class Build:
    pub let plan: BuildPlan
    var command: Command? = none
    ## "" built; otherwise why not.
    pub var failure: str = ""
    pub var done: bool = false
    ## How long luc took (ms), and whether it ran at all.
    pub var seconds: float = 0.0
    var started: int = 0

    pub func init(self, plan: BuildPlan):
        self.plan = plan

    ## Whether it has finished (built or failed); notices the end.
    pub func poll(self) -> bool!:
        if self.done:
            return true
        let command = self.command else return true
        if not command.is_finished():
            return false
        let code = command.exit_code() else -1
        let output = command.output() + command.error_message()
        command.close()
        self.command = none
        self.done = true
        self.seconds = float(time.now() - self.started) / 1000000000.0
        if code != 0:
            self.failure = build_failure(output, self.plan.hint)
            return true
        self.plan.keep() catch failure:
            self.failure = "the built script could not be kept: " + failure.message
        return true

    func begin(self, news: Signal) -> unit!:
        self.started = time.now()
        self.command = self.plan.start(news.waker())

    pub func cancel(self):
        if let command = self.command:
            command.close()
        self.command = none
        self.done = true
        self.failure = "the build was cancelled"

pub class ScriptHosts:
    let builds: map[str, Build]
    let waiting: map[str, ScriptProcess]
    ## The key each Script node (by id) last cooked with: a new key cancels
    ## the old build and stops its waiting child.
    let current: map[int, str]
    var versions: str = ""
    ## The packages the editor was built with (found once).
    var editor: map[str, str]? = none
    var runs: int = 0
    ## Builds started (cache misses) and cooks served, for tests and the status.
    pub var built: int = 0
    pub var cooked: int = 0
    ## Inputs kept between cooks: by "node-port", the content key its file holds;
    ## and how many cooks found theirs kept.
    let kept: map[str, int]
    var inputs: str = ""
    pub var inputs_kept: int = 0
    ## Keys whose entries were marked used this session, and when the cache was
    ## last kept to its limit.
    let marked: map[str, bool]
    var evicted_at: int = 0
    ## Notified when a build or a child process ends or has news (their
    ## commands wake it), and, on the compute worker, when a newer request
    ## arrives: a cook waiting on either blocks on it instead of polling.
    pub let news: Signal

    pub func init(self):
        self.news = Signal.make()
        self.builds = {}
        self.waiting = {}
        self.current = {}
        self.kept = {}
        self.marked = {}

    ## The build for a script (started when it is not built yet); `node` is
    ## the node it is for.
    pub func build(self, node: int, code: str, packages: str, lock: str, folder: str) -> Build!:
        if self.versions == "":
            self.versions = toolchain()
        let editor = self.editor else editor_packages()
        self.editor = editor
        let wanted = script_build.plan(node, code, packages, lock, folder, self.versions, editor)
        if let old = self.current[node]:
            if old != wanted.key:
                self.retire(old)
        self.current[node] = wanted.key
        if let known = self.builds[wanted.key]:
            return known
        let made = Build(wanted)
        self.builds[wanted.key] = made
        # Built in an earlier session: a registry-only build is current by its
        # key; one on checkouts is checked once by luc (a fast no-op build).
        let exists = files.metadata(wanted.program).kind catch failure:
            recover files.FileKind.directory
        if exists == files.FileKind.regular and wanted.pinned():
            made.done = true
            return made
        made.begin(self.news)
        self.built += 1
        return made

    ## A started child of a built key, connected (waiting up to 10 s for it).
    pub func child(self, made: Build) -> ScriptProcess!:
        var process = self.waiting.remove(made.plan.key) else self.start(made)
        if process.finished():
            process.close()
            process = self.start(made)
        var waited = 0
        while not process.connected(20):
            waited += 20
            if process.finished() or waited > 10000:
                let console = process.console()
                process.close()
                error(ui.invalid, "the script process did not start: " + console)
        return process

    ## Start the next child of `made` so the next cook finds it waiting.
    pub func prepare(self, made: Build) -> unit!:
        if made.plan.key in self.waiting:
            return
        self.waiting[made.plan.key] = self.start(made)

    func start(self, made: Build) -> ScriptProcess!:
        return ScriptProcess(made.plan.program, made.plan.folder, self.news.waker())

    ## The file input `port` of node `node` goes to the child in: the one the last
    ## cook wrote when its content key `key` is the same (0: unknown, written
    ## again), else written now.
    pub func input_file(self, node: int, port: int, key: int, data: GeometryData, run: str) -> str!:
        if key == 0:
            let path = paths.join(run, f"in{port}.prism")
            _ = GeometryFile.save(data.geometry, path)
            return path
        if self.inputs == "":
            self.inputs = paths.join(paths.join(scripts_home(), "run"), f"inputs-{time.now()}")
            files.ensure_directory(self.inputs)
        let name = f"{node}-{port}"
        let path = paths.join(self.inputs, name + ".prism")
        if let old = self.kept[name]:
            if old == key and present(path):
                self.inputs_kept += 1
                return path
        _ = self.kept.remove(name)
        _ = GeometryFile.save(data.geometry, path)
        self.kept[name] = key
        return path

    ## A cook is using `made`: its entries are marked used (once a session), and
    ## after a build the cache is kept to its limit (at most every 30 s).
    pub func used(self, made: Build) -> unit!:
        if not (made.plan.key in self.marked):
            script_cache.mark_used(made.plan.folder)
            script_cache.mark_used(made.plan.work)
            self.marked[made.plan.key] = true
        if made.seconds == 0.0 or time.now() - self.evicted_at < 30000000000:
            return
        self.evicted_at = time.now()
        let keep: list[str] = []
        for key in self.current.values():
            if let held = self.builds[key]:
                keep.append(held.plan.folder)
                keep.append(held.plan.work)
        _ = script_cache.evict(keep, script_cache.limit_bytes())

    ## A fresh folder for one cook's files.
    pub func run_folder(self) -> str!:
        self.runs += 1
        let folder = paths.join(paths.join(scripts_home(), "run"), f"{time.now()}-{self.runs}")
        files.ensure_directory(folder)
        return folder

    ## Stop what key `key` holds: its build and its waiting child.
    func retire(self, key: str):
        if let old = self.builds[key]:
            if not old.done:
                old.cancel()
                _ = self.builds.remove(key)
        if let process = self.waiting.remove(key):
            process.kill()
            process.close()

    pub func close(self):
        if self.inputs != "":
            files.remove_tree(self.inputs) catch failure:
                _ = failure
        for process in self.waiting.values():
            process.kill()
            process.close()
        self.waiting.clear()
        for made in self.builds.values():
            if not made.done:
                made.cancel()

## Whether a file is at `path`.
func present(path: str) -> bool:
    let kind = files.metadata(path).kind catch failure:
        return false
    return kind == files.FileKind.regular