dymokomi/luc
Source of release 0.24.0
## Anonymous registry packages: resolve, lock and sync from the static files the
## registry publishes for every release.
## <origin>/<owner>/<name>/versions version SP sha256 SP commit
## <origin>/<owner>/<name>/<version>.prisma that release's package.prisma
## <origin>/<owner>/<name>/<version>.pack history-free Git pack of the source
## No account, credential or signature is involved. luc.lock pins every package to the
## SHA-256 of its pack, and a download whose digest differs is refused.
import c
from luce_std import files
from luce_git import git
import memory
import os
from luce_std import paths
import strings
from luce_pkg import pkg
from luce_prism import prism
import apps
import checkout
import progress
import project_file
import registry_http
import thread
import time
pub let invalid = pkg.invalid
pub let lock_name = "luc.lock"
pub let max_packages: usize = 256
let max_pack: usize = 67108864
let max_listing: usize = 4194304
## The registry to use; LUC_REGISTRY overrides it with loopback HTTP for tests.
pub func origin() -> str:
return os.env((c.str)"LUC_REGISTRY") else registry_http.production_origin
pub struct Locked:
pub var name: str
pub var owner: str
pub var version: str
pub var sha256: str
pub var commit: str
## Owns every string of a resolved or decoded graph.
pub struct Graph:
var arena: u8[]
var used: usize
pub var packages: Locked[]
pub var count: usize
pub func close():
if self.arena.length > 0: free(self.arena) in memory.heap
if self.packages.length > 0: free(self.packages) in memory.heap
self = Graph()
func keep(text: str) -> str!:
if self.used + text.length > self.arena.length: error(invalid, "dependency graph text exceeds its bound")
let start = self.used
for i in 0..<text.length: self.arena[start + i] = text.bytes[i]
self.used += text.length
return (str)self.arena[start..<self.used]
func find(name: str) -> usize?:
for i in 0..<self.count:
if self.packages[i].name == name: return i
return none
func new_graph() -> Graph!:
var graph: Graph
errdefer graph.close()
graph.arena = try new u8[1048576] --- in memory.heap
graph.packages = try new Locked[max_packages] --- in memory.heap
return graph
## GET one public file. Anything but 200 is an error naming what was asked for.
func fetch(target: str, output: u8[]) -> usize!:
let response = try registry_http.request(origin(), "GET", target, "", "", b"", output)
if response.status == 404: error(invalid, "the registry has no such package or version")
if response.status != 200: error(invalid, "the registry request failed")
return response.count
func field(line: str, index: usize) -> str:
var start: usize
var seen: usize
for i in 0..<line.length + 1:
if i != line.length and line.bytes[i] != ' ': continue
if seen == index: return (str)line.bytes[start..<i]
seen += 1
start = i + 1
return ""
func newer(left: pkg.Version, right: pkg.Version) -> bool:
if left.major != right.major: return left.major > right.major
if left.minor != right.minor: return left.minor > right.minor
return left.patch > right.patch
## `^1.2.3` accepts the same major from 1.2.3 up; `^0.2.3` the same minor; `^0.0.3` and
## a bare version accept exactly that version. No requirement accepts every release, so
## the newest is chosen and luc.lock keeps it.
func satisfies(version: pkg.Version, requirement: str) -> bool!:
if requirement.length == 0: return true
let caret = requirement.bytes[0] == '^'
let wanted = try pkg.parse_version((str)requirement.bytes[1..] if caret else requirement)
let same = version.major == wanted.major and version.minor == wanted.minor and version.patch == wanted.patch
if not caret: return same
if same: return true
if not newer(version, wanted): return false
if wanted.major > 0: return version.major == wanted.major
if wanted.minor > 0: return version.major == 0 and version.minor == wanted.minor
return false
## The `versions` line of the newest release that satisfies `requirement`.
func select(listing: str, requirement: str) -> str!:
var best = ""
var best_version: pkg.Version
var start: usize
for end in 0..<listing.length:
if listing.bytes[end] != '\n': continue
let line = (str)listing.bytes[start..<end]
start = end + 1
let version = pkg.parse_version(field(line, 0)) catch: continue
if not try satisfies(version, requirement): continue
if best.length == 0 or newer(version, best_version):
best = line
best_version = version
if best.length == 0: error(invalid, "no released version satisfies the requirement")
return best
## The newest released version of a package, for `luc add [owner/]name`.
pub func latest(owner: str, name: str, output: u8[]) -> str!:
let listing = try new u8[max_listing] --- in memory.heap
defer free(listing) in memory.heap
var target: u8[256]
let size = try fetch(try strings.format(target, f"/{owner}/{name}/versions"), listing)
var best = ""
var best_version: pkg.Version
var start: usize
for end in 0..<size:
if listing[end] != '\n': continue
let text = field((str)listing[start..<end], 0)
start = end + 1
let version = pkg.parse_version(text) catch: continue
if best.length == 0 or newer(version, best_version):
best = text
best_version = version
if best.length == 0: error(invalid, "the package has no released version")
return try strings.format(output, f"{best}")
## The owner of the one registry package called `name`, copied into `output`, for
## `luc install name`. The registry's index lists each package on a line that starts
## `owner/name` and a tab. No package of that name is an error, and so are several;
## the error's text is written into `message`, which must outlive the error, and
## names every candidate as `owner/name`.
pub func owner_of(name: str, output: u8[], message: u8[]) -> str!:
let listing = try new u8[max_listing] --- in memory.heap
defer free(listing) in memory.heap
let size = try fetch("/index", listing)
var owner = ""
var matches: usize
var written = (try strings.format(message, f"several registry packages are called {name}:")).length
var start: usize
for end in 0..<size:
if listing[end] != '\n': continue
let line = (str)listing[start..<end]
start = end + 1
var stop: usize
while stop < line.length and line.bytes[stop] != '\t': stop += 1
let coordinate = (str)line.bytes[..<stop]
let (candidate, package) = strings.split_once(coordinate, "/") else continue
if package != name: continue
matches += 1
if matches == 1: owner = try strings.format(output, f"{candidate}")
if written + coordinate.length + 2 <= message.length:
written += (try strings.format(message[written..], f" {coordinate}")).length
if matches == 0: error(invalid, try strings.format(message, f"no registry package is called {name}"))
if matches > 1:
let hint = "; name one as owner/name"
if written + hint.length <= message.length: written += (try strings.format(message[written..], f"{hint}")).length
error(invalid, (str)message[..<written])
return owner
## Select one release for a requirement and walk its own dependencies.
# mark: Resolving ==============================================================================
## A registry package some package asks for, with the version it will take.
struct Wanted:
var owner: str
var name: str
var requirement: str
## The registry packages asked for so far, their text kept in the graph.
struct WantedList:
var items: Wanted[]
var count: usize
func add(graph: Graph*, owner: str, name: str, requirement: str) -> !:
if self.count >= self.items.length: error(invalid, "too many dependency requirements")
self.items[self.count] = Wanted(owner = try graph.keep(owner), name = try graph.keep(name), requirement = try graph.keep(requirement))
self.count += 1
func wanted_list() -> WantedList!:
return WantedList(items = try new Wanted[max_packages * 4] --- in memory.heap, count = 0)
## One package looked up on the registry: the versions line its requirement selects and
## that version's definition, or why the lookup failed. Both live on the heap.
struct Lookup:
var owner: str
var name: str
var requirement: str
var line: str
var document: u8[]
var length: usize
var failure: str
## The lookups of one level, shared by the threads that make them.
struct Lookups:
var items: Lookup[]
var count: usize
var next: @u64
## Lookups run on this many threads at once: each is two requests, so a level of the
## graph costs about one round trip's worth of waiting rather than one per package.
let lookup_threads: usize = 8
func look_up(item: Lookup*) -> !:
let listing = try new u8[max_listing] ---
defer free(listing)
var target: u8[256]
let size = try fetch(try strings.format(target, f"/{item.owner}/{item.name}/versions"), listing)
item.line = try strings.copy(try select((str)listing[..<size], item.requirement))
item.document = try new u8[65536] ---
item.length = try fetch(try strings.format(target, f"/{item.owner}/{item.name}/{field(item.line, 0)}.prisma"), item.document)
func lookup_worker(raw: void*?):
let shared = (Lookups*)(raw else return)
with memory.heap:
while true:
let index = (usize)shared.next.add(1, .relaxed)
if index >= shared.count: return
let item = &shared.items[index]
look_up(item) catch failure:
item.failure = strings.copy(failure.message) catch ignored: recover "a registry lookup failed"
## Make every lookup, on threads when there is more than one.
func look_up_all(shared: Lookups*) -> !:
if shared.count == 1:
lookup_worker((void*)shared)
return
var workers: thread.Handle[8]
var started: usize
while started < lookup_threads and started < shared.count:
workers[started] = thread.spawn(lookup_worker, (void*)shared, name = "luc resolve") catch failure:
break
started += 1
if started == 0:
lookup_worker((void*)shared)
for index in 0..<started:
try workers[index].join()
func release_lookups(shared: Lookups*):
with memory.heap:
for item in shared.items[..<shared.count]:
if item.line.length > 0: strings.release(item.line)
if item.document.length > 0: free(item.document)
if item.failure.length > 0: strings.release(item.failure)
free(shared.items)
## Resolve the registry packages `first` asks for and everything they depend on, one level
## of the graph at a time: a level's new packages are looked up together, then taken in
## the order they were asked for, so the lock is what a one-by-one walk would write.
func resolve_levels(graph: Graph*, first: WantedList) -> !:
var level = first
var depth: usize
while level.count > 0:
if depth > 64: error(invalid, "dependencies nest too deeply")
let asked = level.items
defer free(asked) in memory.heap
var shared = Lookups(items = try new Lookup[level.count] --- in memory.heap, count = 0)
defer release_lookups(&shared)
for wanted in level.items[..<level.count]:
if graph.find(wanted.name) != none: continue
var seen = false
for other in shared.items[..<shared.count]:
if other.name == wanted.name: seen = true
if not seen:
shared.items[shared.count] = Lookup(owner = wanted.owner, name = wanted.name, requirement = wanted.requirement, line = "", length = 0, failure = "")
shared.count += 1
if shared.count > 0:
try look_up_all(&shared)
var next = try wanted_list()
for wanted in level.items[..<level.count]:
if let existing = graph.find(wanted.name):
let chosen = graph.packages[existing]
if chosen.owner != wanted.owner: error(invalid, "two dependencies share a package name but not an owner")
if not try satisfies(try pkg.parse_version(chosen.version), wanted.requirement):
var message: u8[512]
error(invalid, try graph.keep(try strings.format(message, f"{wanted.owner}/{wanted.name} {chosen.version} was chosen for one dependency, but another needs {wanted.requirement}; they must agree on one version")))
continue
var found: Lookup* = &shared.items[0]
for index in 0..<shared.count:
if shared.items[index].name == wanted.name: found = &shared.items[index]
if found.failure.length > 0: error(invalid, try graph.keep(found.failure))
if graph.count >= max_packages: error(invalid, "too many packages in the dependency graph")
let version = try graph.keep(field(found.line, 0))
graph.packages[graph.count] = Locked(name = wanted.name, owner = wanted.owner, version = version,
sha256 = try graph.keep(field(found.line, 1)), commit = try graph.keep(field(found.line, 2)))
graph.count += 1
var definition = try pkg.decode_package(found.document[..<found.length])
defer definition.close()
if definition.owner != wanted.owner or definition.name != wanted.name or definition.version != version:
error(invalid, "the registry served a definition for a different package")
# A released package's `path` is its author's local override; consumers use the registry.
for i in 0..<definition.dependency_count:
let item = try definition.dependency_at(i)
try next.add(graph, item.owner, item.name, item.version)
level = next
depth += 1
free(level.items) in memory.heap
## Walk a local package: path dependencies are followed on disk, the registry ones
## collected for `resolve_levels`. `registry` marks a package that itself came from the
## registry: its `path` overrides are its author's local checkouts and are ignored. So
## is a checkout path that is not there, as in a fresh clone of an application whose
## author keeps its packages as siblings: the registry release its owner and version
## name stands in, as the compilers then look in .luc/deps for it.
func resolve_local(graph: Graph*, directory: str, depth: usize, wanted: WantedList*, registry: bool = false) -> !:
if depth > 32: error(invalid, "path dependencies nest too deeply or form a cycle")
var definition = try project_file.read(directory)
defer definition.close()
for i in 0..<definition.dependency_count:
let item = try definition.dependency_at(i)
if item.path.length == 0 or registry:
try wanted.add(graph, item.owner, item.name, item.version)
continue
let location = try paths.join(directory, item.path)
defer location.release()
if not try project_file.exists_in(location.value):
try wanted.add(graph, item.owner, item.name, item.version)
continue
try resolve_local(graph, location.value, depth + 1, wanted)
func write_lock(root: str, graph: Graph*) -> !:
let buffer = try new u8[1048576] --- in memory.heap
defer free(buffer) in memory.heap
var used: usize
var line: u8[512]
let head = try strings.format(line, f"#prisma 4.0\ndef lock \"luc\" {{\n str origin = \"{origin()}\"\n")
for i in 0..<head.length: buffer[used + i] = head.bytes[i]
used += head.length
for index in 0..<graph.count:
let item = graph.packages[index]
let entry = try strings.format(line, f" def package \"{item.name}\" {{\n str owner = \"{item.owner}\"\n str version = \"{item.version}\"\n str sha256 = \"{item.sha256}\"\n str commit = \"{item.commit}\"\n }}\n")
if used + entry.length + 2 > buffer.length: error(invalid, "luc.lock exceeds its bound")
for i in 0..<entry.length: buffer[used + i] = entry.bytes[i]
used += entry.length
buffer[used] = '}'
buffer[used + 1] = '\n'
used += 2
let path = try paths.join(root, lock_name)
defer path.release()
try files.write_atomic((c.str)path.value, buffer[..<used])
## `luc lock`: resolve the whole graph against the registry and write luc.lock.
pub func lock(root: str, registry: bool = false) -> usize!:
var graph = try new_graph()
defer graph.close()
var wanted = try wanted_list()
try resolve_local(&graph, root, 0, &wanted, registry)
try resolve_levels(&graph, wanted)
try write_lock(root, &graph)
return graph.count
func hex_valid(text: str, length: usize) -> bool:
if text.length != length: return false
for byte in text.bytes:
if not ((byte >= '0' and byte <= '9') or (byte >= 'a' and byte <= 'f')): return false
return true
## Copies the property into the graph before the Prism value is released.
func text_of(graph: Graph*, document: prism.Document*, path: str, name: str) -> str!:
let value = try document.get(path, name)
defer value.release()
return try graph.keep(try value.value().text_at())
pub func read_lock(root: str) -> Graph!:
let path = try paths.join(root, lock_name)
defer path.release()
if not files.exists((c.str)path.value): error(invalid, "no luc.lock here; run 'luc lock' first")
let reference = try prism.Document.load(path.value)
defer reference.release()
let document = reference.value()
var graph = try new_graph()
errdefer graph.close()
for index in 0..<document.element_count():
let element = try document.path_at(index)
if try document.type_at(element) != "package": continue
if graph.count >= max_packages: error(invalid, "luc.lock names too many packages")
var last: usize
for i in 0..<element.length:
if element.bytes[i] == '/': last = i
let item = Locked(name = try graph.keep((str)element.bytes[last + 1..]),
owner = try text_of(&graph, document, element, "owner"),
version = try text_of(&graph, document, element, "version"),
sha256 = try text_of(&graph, document, element, "sha256"),
commit = try text_of(&graph, document, element, "commit"))
_ = try pkg.parse_version(item.version)
if not hex_valid(item.sha256, 64) or not hex_valid(item.commit, 40): error(invalid, "luc.lock has a malformed digest or commit")
graph.packages[graph.count] = item
graph.count += 1
return graph
func ensure_directory(path: str) -> !:
let terminated = try strings.copy(path)
defer strings.release(terminated)
if not files.exists((c.str)terminated): try files.create_directory((c.str)terminated)
## <luc home>/cache/<owner>/<name>, created as needed. LUC_CACHE overrides the root.
func cache_directory(owner: str, name: str) -> str!:
var root: str
if let chosen = os.env((c.str)"LUC_CACHE"):
root = try strings.copy(chosen)
else:
# luc's own home: ~/.luce, %USERPROFILE%\.luce on Windows, or LUC_HOME
let base = try apps.home()
defer strings.release(base)
root = try apps.child(base, "cache")
defer strings.release(root)
try ensure_directory(root)
let group = try apps.child(root, owner)
defer strings.release(group)
return try apps.child(group, name)
func digest_matches(bytes: const u8[], expected: str) -> bool!:
var digest: u8[32]
_ = try pkg.source_digest(bytes, digest)
let digits = "0123456789abcdef"
for i in 0..<(usize)32:
if expected.bytes[i * 2] != digits.bytes[(usize)(digest[i] >> 4)]: return false
if expected.bytes[i * 2 + 1] != digits.bytes[(usize)(digest[i] & 15)]: return false
return true
## The verified pack of one locked package: from the cache, else downloaded into it.
## With `offline`, a package missing from the cache is an error and nothing is fetched.
func verified_pack(item: Locked, offline: bool, output: u8[]) -> usize!:
let directory = try cache_directory(item.owner, item.name)
defer strings.release(directory)
var name: u8[96]
let cached = try paths.join(directory, try strings.format(name, f"{item.version}.pack"))
defer cached.release()
if files.exists((c.str)cached.value):
let bytes = try files.read((c.str)cached.value, limit = max_pack)
defer bytes.release()
if try digest_matches(bytes.value, item.sha256):
for i in 0..<bytes.value.length: output[i] = bytes.value[i]
return bytes.value.length
if offline: error(invalid, "a cached package does not match luc.lock")
elif offline:
error(invalid, "a locked package is not in the cache; run 'luc sync' online once")
var target: u8[256]
let started = time.now()
let size = try fetch(try strings.format(target, f"/{item.owner}/{item.name}/{item.version}.pack"), output)
try progress.detail(f"downloaded {item.owner}/{item.name} {item.version} ({(size + 1023) // 1024} KB, {progress.seconds_since(started)}s)")
if not try digest_matches(output[..<size], item.sha256):
error(invalid, "a downloaded package does not match the SHA-256 in luc.lock")
try files.write_atomic((c.str)cached.value, output[..<size])
return size
## Download one released package and unpack it into `destination`, which must not
## exist. `requirement` is a version, a caret requirement, or empty for the newest.
## The pack is checked against the SHA-256 the registry lists for that version. Its
## development paths (tests/, dev/, …) are left out unless `development`.
pub func download(owner: str, name: str, requirement: str, destination: str, version_output: u8[], development: bool = false) -> str!:
var graph = try new_graph()
defer graph.close()
let listing = try new u8[max_listing] --- in memory.heap
defer free(listing) in memory.heap
var target: u8[256]
let size = try fetch(try strings.format(target, f"/{owner}/{name}/versions"), listing)
var newest: u8[64]
let line = try select((str)listing[..<size], requirement if requirement.length > 0 else try latest(owner, name, newest))
let item = Locked(name = try graph.keep(name), owner = try graph.keep(owner), version = try graph.keep(field(line, 0)),
sha256 = try graph.keep(field(line, 1)), commit = try graph.keep(field(line, 2)))
if not hex_valid(item.sha256, 64) or not hex_valid(item.commit, 40): error(invalid, "the registry listing is malformed")
let bytes = try new u8[max_pack] --- in memory.heap
defer free(bytes) in memory.heap
let length = try verified_pack(item, false, bytes)
var pack = try git.decode_pack(bytes[..<length])
defer pack.close()
var commit: u8[20]
try git.decode_id(item.commit, commit)
try checkout.materialize(&pack, commit, destination, history = false, development = development)
return try strings.format(version_output, f"{item.version}")
## True when this package, or a path dependency of it, needs a registry package.
func needs_registry(directory: str, depth: usize, registry: bool = false) -> bool!:
if depth > 32: return false
var definition = try project_file.read(directory)
defer definition.close()
for i in 0..<definition.dependency_count:
let item = try definition.dependency_at(i)
if item.path.length == 0 or registry: return true
let location = try paths.join(directory, item.path)
defer location.release()
# A missing checkout comes from the registry, as resolve_local takes it.
if not try project_file.exists_in(location.value): return true
if try needs_registry(location.value, depth + 1): return true
return false
## Before a build: lock if nothing is locked yet, then make .luc/deps match the lock.
## A project with only path dependencies never touches the network.
pub func prepare(root: str, registry: bool = false) -> !:
if not try needs_registry(root, 0, registry): return
let path = try paths.join(root, lock_name)
defer path.release()
if not files.exists((c.str)path.value):
try progress.step(f"resolving dependency versions")
_ = try lock(root, registry)
_ = try sync(root)
# mark: Downloading in parallel ===============================================================
## Packs download on this many threads at once, each with a pack buffer of its own.
let download_threads: usize = 4
## The packages a prefetch downloads, shared by its workers: each takes the next index.
struct Prefetch:
var items: Locked[]
var next: @u64
var failures: @u64
var message: str
## Download every pack of `items` the cache lacks, on several threads, so a fresh install
## waits for its slowest download rather than the sum of them. The packs land in the cache,
## where the ordered unpacking reads them; a failed download is reported there again.
func prefetch(items: Locked[]) -> !:
var missing: usize
for item in items:
if not try in_cache(item): missing += 1
if missing < 2: return
try progress.step(f"fetching {missing} packages")
var shared = Prefetch(items = items, message = "")
var workers: thread.Handle[4]
var started: usize
while started < download_threads and started < missing:
workers[started] = thread.spawn(prefetch_worker, (void*)&shared, name = "luc fetch") catch failure:
break
started += 1
for index in 0..<started:
try workers[index].join()
func prefetch_worker(raw: void*?):
let shared = (Prefetch*)(raw else return)
let bytes = new u8[max_pack] --- in memory.heap catch failure:
_ = shared.failures.add(1, .relaxed)
return
defer free(bytes) in memory.heap
while true:
let index = (usize)shared.next.add(1, .relaxed)
if index >= shared.items.length: return
let item = shared.items[index]
let present = in_cache(item) catch failure: recover false
if present: continue
_ = verified_pack(item, false, bytes) catch failure:
_ = shared.failures.add(1, .relaxed)
recover 0
## Whether the cache already holds this package's pack.
func in_cache(item: Locked) -> bool!:
let directory = try cache_directory(item.owner, item.name)
defer strings.release(directory)
var name: u8[96]
let path = try paths.join(directory, try strings.format(name, f"{item.version}.pack"))
defer path.release()
return files.exists((c.str)path.value)
## `luc sync`: make .luc/deps match luc.lock exactly, from the cache or the registry.
pub func sync(root: str, offline: bool = false) -> usize!:
var graph = try read_lock(root)
defer graph.close()
if not offline:
try prefetch(graph.packages[..<graph.count])
let hidden = try paths.join(root, ".luc")
defer hidden.release()
try ensure_directory(hidden.value)
let parent = try paths.join(hidden.value, "deps")
defer parent.release()
try ensure_directory(parent.value)
let bytes = try new u8[max_pack] --- in memory.heap
defer free(bytes) in memory.heap
var announced = false
for index in 0..<graph.count:
let item = graph.packages[index]
let destination = try project_file.dependency_directory(root, item.name)
defer strings.release(destination)
var marker_name: u8[96]
let marker = try paths.join(destination, try strings.format(marker_name, f".luc-{item.sha256}"))
defer marker.release()
if files.exists((c.str)marker.value): continue
if not announced:
try progress.step(f"unpacking {graph.count} dependencies")
announced = true
let size = try verified_pack(item, offline, bytes)
var pack = try git.decode_pack(bytes[..<size])
defer pack.close()
var commit: u8[20]
try git.decode_id(item.commit, commit)
let destination_z = try strings.copy(destination)
defer strings.release(destination_z)
if files.exists((c.str)destination_z): try files.remove_tree((c.str)destination_z)
# A dependency is built, never developed: its tests and dev paths stay out.
try checkout.materialize(&pack, commit, destination, history = false, development = false)
try files.write((c.str)marker.value, b"")
return graph.count