dymokomi/luced-3d
Source of release 0.11.8
0.11.8 / src / script_hosts.luc
## The Script nodes' builds and processes, kept for the app's life by the
## SourceCache the compute worker (and a local network) cooks with. A build
## runs in the background under its key; a cook that is superseded while it
## waits leaves it running, and the next cook of that key waits for the same
## build. Each key keeps one child started and waiting, so a cook pays no
## start-up: the child serves one cook and leaves (each cook starts fresh, as
## Houdini's Python SOP does), and the next is started at once. An input whose
## content key (its node's stamp) is the one the last cook sent keeps its file.
## The disk cache is kept to its limit (script_cache.luc).
from luce_std import files, paths
from luce_geocore.geocore import GeometryFile
from geometry_data import GeometryData
import script_cache
from luce_std.process import Command
from luce_geocore.script_host import ScriptProcess
from script_build import BuildPlan, toolchain, build_failure, scripts_home
from script_packages import editor_packages
import script_build
from luce_ui import ui
from signal import Signal
import time
## A build in flight or done.
pub class Build:
pub let plan: BuildPlan
var command: Command? = none
## "" built; otherwise why not.
pub var failure: str = ""
pub var done: bool = false
## How long luc took (ms), and whether it ran at all.
pub var seconds: float = 0.0
var started: int = 0
pub func init(self, plan: BuildPlan):
self.plan = plan
## Whether it has finished (built or failed); notices the end.
pub func poll(self) -> bool!:
if self.done:
return true
let command = self.command else return true
if not command.is_finished():
return false
let code = command.exit_code() else -1
let output = command.output() + command.error_message()
command.close()
self.command = none
self.done = true
self.seconds = float(time.now() - self.started) / 1000000000.0
if code != 0:
self.failure = build_failure(output, self.plan.hint)
return true
self.plan.keep() catch failure:
self.failure = "the built script could not be kept: " + failure.message
return true
func begin(self, news: Signal) -> unit!:
self.started = time.now()
self.command = self.plan.start(news.waker())
pub func cancel(self):
if let command = self.command:
command.close()
self.command = none
self.done = true
self.failure = "the build was cancelled"
pub class ScriptHosts:
let builds: map[str, Build]
let waiting: map[str, ScriptProcess]
## The key each Script node (by id) last cooked with: a new key cancels
## the old build and stops its waiting child.
let current: map[int, str]
var versions: str = ""
## The packages the editor was built with (found once).
var editor: map[str, str]? = none
var runs: int = 0
## Builds started (cache misses) and cooks served, for tests and the status.
pub var built: int = 0
pub var cooked: int = 0
## Inputs kept between cooks: by "node-port", the content key its file holds;
## and how many cooks found theirs kept.
let kept: map[str, int]
var inputs: str = ""
pub var inputs_kept: int = 0
## Keys whose entries were marked used this session, and when the cache was
## last kept to its limit.
let marked: map[str, bool]
var evicted_at: int = 0
## Notified when a build or a child process ends or has news (their
## commands wake it), and, on the compute worker, when a newer request
## arrives: a cook waiting on either blocks on it instead of polling.
pub let news: Signal
pub func init(self):
self.news = Signal.make()
self.builds = {}
self.waiting = {}
self.current = {}
self.kept = {}
self.marked = {}
## The build for a script (started when it is not built yet); `node` is
## the node it is for.
pub func build(self, node: int, code: str, packages: str, lock: str, folder: str) -> Build!:
if self.versions == "":
self.versions = toolchain()
let editor = self.editor else editor_packages()
self.editor = editor
let wanted = script_build.plan(node, code, packages, lock, folder, self.versions, editor)
if let old = self.current[node]:
if old != wanted.key:
self.retire(old)
self.current[node] = wanted.key
if let known = self.builds[wanted.key]:
return known
let made = Build(wanted)
self.builds[wanted.key] = made
# Built in an earlier session: a registry-only build is current by its
# key; one on checkouts is checked once by luc (a fast no-op build).
let exists = files.metadata(wanted.program).kind catch failure:
recover files.FileKind.directory
if exists == files.FileKind.regular and wanted.pinned():
made.done = true
return made
made.begin(self.news)
self.built += 1
return made
## A started child of a built key, connected (waiting up to 10 s for it).
pub func child(self, made: Build) -> ScriptProcess!:
var process = self.waiting.remove(made.plan.key) else self.start(made)
if process.finished():
process.close()
process = self.start(made)
var waited = 0
while not process.connected(20):
waited += 20
if process.finished() or waited > 10000:
let console = process.console()
process.close()
error(ui.invalid, "the script process did not start: " + console)
return process
## Start the next child of `made` so the next cook finds it waiting.
pub func prepare(self, made: Build) -> unit!:
if made.plan.key in self.waiting:
return
self.waiting[made.plan.key] = self.start(made)
func start(self, made: Build) -> ScriptProcess!:
return ScriptProcess(made.plan.program, made.plan.folder, self.news.waker())
## The file input `port` of node `node` goes to the child in: the one the last
## cook wrote when its content key `key` is the same (0: unknown, written
## again), else written now.
pub func input_file(self, node: int, port: int, key: int, data: GeometryData, run: str) -> str!:
if key == 0:
let path = paths.join(run, f"in{port}.prism")
_ = GeometryFile.save(data.geometry, path)
return path
if self.inputs == "":
self.inputs = paths.join(paths.join(scripts_home(), "run"), f"inputs-{time.now()}")
files.ensure_directory(self.inputs)
let name = f"{node}-{port}"
let path = paths.join(self.inputs, name + ".prism")
if let old = self.kept[name]:
if old == key and present(path):
self.inputs_kept += 1
return path
_ = self.kept.remove(name)
_ = GeometryFile.save(data.geometry, path)
self.kept[name] = key
return path
## A cook is using `made`: its entries are marked used (once a session), and
## after a build the cache is kept to its limit (at most every 30 s).
pub func used(self, made: Build) -> unit!:
if not (made.plan.key in self.marked):
script_cache.mark_used(made.plan.folder)
script_cache.mark_used(made.plan.work)
self.marked[made.plan.key] = true
if made.seconds == 0.0 or time.now() - self.evicted_at < 30000000000:
return
self.evicted_at = time.now()
let keep: list[str] = []
for key in self.current.values():
if let held = self.builds[key]:
keep.append(held.plan.folder)
keep.append(held.plan.work)
_ = script_cache.evict(keep, script_cache.limit_bytes())
## A fresh folder for one cook's files.
pub func run_folder(self) -> str!:
self.runs += 1
let folder = paths.join(paths.join(scripts_home(), "run"), f"{time.now()}-{self.runs}")
files.ensure_directory(folder)
return folder
## Stop what key `key` holds: its build and its waiting child.
func retire(self, key: str):
if let old = self.builds[key]:
if not old.done:
old.cancel()
_ = self.builds.remove(key)
if let process = self.waiting.remove(key):
process.kill()
process.close()
pub func close(self):
if self.inputs != "":
files.remove_tree(self.inputs) catch failure:
_ = failure
for process in self.waiting.values():
process.kill()
process.close()
self.waiting.clear()
for made in self.builds.values():
if not made.done:
made.cancel()
## Whether a file is at `path`.
func present(path: str) -> bool:
let kind = files.metadata(path).kind catch failure:
return false
return kind == files.FileKind.regular