dymokomi/luce-pkg-server
Source of release 0.6.1
0.6.1 / src / repositories.lucb
## Internal repository storage. Principal MUST come from verified authentication,
## never from a request field. Ref updates require bounded graph closure; object
## uploads alone do not validate graph edges or publish refs.
from luce_db import db
import c
from luce_std import files, paths
from luce_prism import prism
import interop
from luce_git import git
import strings
import memory
from luce_pkg import pkg
import time
from luce_std import utf8
import site
import pages
import objects as object_files
pub let max_object_bytes: usize = 67108864
## A Prism commit is durable on its own: one write-ahead-log transaction. A bake folds
## the log into a new snapshot, in time proportional to the whole store, and holds the
## single writer slot meanwhile. So requests never bake per operation. `committed`
## counts the journal bytes this process wrote since its last bake and bakes once they
## pass `bake_threshold`, well under the journal's 256 MiB cap; the deployment also
## bakes on a timer (`luce-pkg-admin checkpoint <socket>`), covering account writes.
## A writer that meets a running bake queues for up to `commit_wait_ms` instead of
## failing; the HTTP deadlines are minutes.
pub let commit_wait_ms: i64 = 60000
pub let bake_threshold: u64 = 33554432
var unbaked: @u64
## Record `bytes` of committed journal and bake when the unbaked tail passes the
## threshold. One caller claims each bake. A bake that meets another writer or bake is
## left to the next threshold or the timer; the commits are already durable.
func committed(database: db.Database*, bytes: usize) -> !:
let total = unbaked.add((u64)bytes, .relaxed) + (u64)bytes
if total < bake_threshold: return
if unbaked.swap(0, .acq_rel) < bake_threshold: return
_ = database.bake(wait_ms = commit_wait_ms) catch failure:
if failure.code == prism.busy or failure.code == prism.timed_out:
recover -1
error(failure.code, failure.message)
## Attempts at a transaction whose commit met a concurrent write. The transaction is
## rebuilt from a fresh snapshot each time, so every check it makes is repeated.
let commit_attempts: usize = 8
## Commit `tx`, answering false when a concurrent commit overlapped its writes.
func commit_unless_raced(tx: prism.Session*) -> bool!:
_ = tx.commit(wait_ms = commit_wait_ms) catch failure:
if failure.code == prism.conflict: return false
error(failure.code, failure.message)
return true
func valid_owner(value: str) -> bool:
if value.length == 0 or value.length > 32: return false
# Names the public site uses for its own pages and routes.
for reserved in ["applications", "tools", "packages", "assets", "index", "git", "v1", "health"]:
if value == reserved: return false
for byte in value.bytes:
if not ((byte >= 'a' and byte <= 'z') or (byte >= '0' and byte <= '9') or byte == '_'): return false
return true
pub func valid_name(value: str) -> bool:
if value.length == 0 or value.length > 64: return false
for i in 0..<value.length:
let byte = value.bytes[i]
let alnum = (byte >= 'a' and byte <= 'z') or (byte >= '0' and byte <= '9')
if not alnum and (i == 0 or (byte != '_' and byte != '-')): return false
return true
func join(parts: const str[], output: u8[]) -> str!:
var at: usize = 0
for part in parts:
if part.length > output.length - at: error(prism.invalid, "repository path too long")
for byte in part.bytes:
output[at] = byte
at += 1
return (str)output[..<at]
func repo_path(principal: str, owner: str, name: str, output: u8[]) -> str!:
if not valid_owner(owner) or not valid_name(name) or principal != owner:
error(prism.invalid, "invalid repository identity or name")
# Prism names are identifiers, whereas package names may contain hyphens.
# Encode every byte, rather than lossy escaping that could alias two names.
let digits = b"0123456789abcdef"
var encoded_name: u8[128]
for i in 0..<name.length:
encoded_name[i * 2] = digits[(usize)(name.bytes[i] >> 4)]
encoded_name[i * 2 + 1] = digits[(usize)(name.bytes[i] & 15)]
return try join(["/repositories/", owner, "/", (str)encoded_name[..<name.length * 2]], output)
func same(left: const u8[], right: const u8[]) -> bool:
if left.length != right.length: return false
for i in 0..<left.length:
if left[i] != right[i]: return false
return true
func require_owner(session: prism.Session*, path: str, principal: str) -> !:
let value = try session.get(path, "owner")
defer value.release()
if try value.value().text_at() != principal:
error(prism.invalid, "repository owner mismatch")
## Keep Git object bytes in `<database>.objects` (see objects.lucb). Call once at
## startup, before any worker thread, in every process that reads or writes objects.
pub func use_objects(database_path: str) -> !:
try object_files.configure(database_path)
## Invoke once on the database owner before starting concurrent workers.
pub func initialize(database: db.Database*) -> !:
let tx = try database.begin()
defer tx.release()
if try tx.value().contains("/repositories"): return
try tx.value().add("/repositories", "directory")
_ = try tx.value().commit(wait_ms = 1000)
_ = try database.bake(wait_ms = 1000)
## Creates a private repository for an existing account. Duplicate creation is a
## conflict, never replacement. A concurrency conflict must be retried by caller.
pub func create(database: db.Database*, principal: str, name: str) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, principal, name, path_buffer)
var user_buffer: u8[64]
let user = try join(["/users/", principal], user_buffer)
var owner_buffer: u8[64]
let owner_path = try join(["/repositories/", principal], owner_buffer)
var objects_buffer: u8[208]
let objects_path = try join([path, "/objects"], objects_buffer)
let tx = try database.begin()
defer tx.release()
if not try tx.value().contains(user): error(prism.invalid, "repository account does not exist")
if not try tx.value().contains("/repositories"): error(prism.invalid, "repository storage is not initialized")
if try tx.value().contains(path): error(prism.conflict, "repository already exists")
if not try tx.value().contains(owner_path): try tx.value().add(owner_path, "directory")
try tx.value().add(path, "repository")
try tx.value().add(objects_path, "directory")
let owner = try prism.Value.text(principal)
defer owner.release()
try tx.value().set(path, "owner", owner)
let label = try prism.Value.text(name)
defer label.release()
try tx.value().set(path, "name", label)
_ = try tx.value().commit(wait_ms = commit_wait_ms)
try committed(database, path.length * 3 + 256)
func object_path(path: str, id: const u8[], output: u8[]) -> str!:
var encoded_id: u8[40]
_ = try git.hex_id(id, encoded_id)
return try join([path, "/objects/", (str)encoded_id], output)
## What a push stores for one object: its id, kind and envelope size. The bytes are
## already durable in the object files when this is listed in the database.
struct Listing:
var id: u8[20]
var kind: str
var size: usize
## Objects listed per transaction: far below Prism's per-layer operation limit, and
## small enough that a large push commits in steady steps.
let listing_batch: usize = 1024
## List objects whose files are durable. A listing that already exists is left as it is:
## the file store has already checked that its bytes are identical.
func list_objects(database: db.Database*, path: str, principal: str, listings: const Listing[]) -> !:
var at: usize = 0
while at < listings.length:
let end = at + listing_batch if listings.length - at > listing_batch else listings.length
var listed = false
for attempt in 0..<commit_attempts:
if try list_batch(database, path, principal, listings[at..<end]):
listed = true
break
if not listed: error(prism.conflict, "object listing kept meeting concurrent writes")
at = end
## One transaction of `list_objects`; false when its commit met a concurrent write, for
## instance another push listing the same object.
func list_batch(database: db.Database*, path: str, principal: str, listings: const Listing[]) -> bool!:
let tx = try database.begin()
defer tx.release()
try require_owner(tx.value(), path, principal)
var journal: usize = 0
for listing in listings:
var object_buffer: u8[256]
let location = try object_path(path, listing.id, object_buffer)
if try tx.value().contains(location): continue
try tx.value().add(location, "git-object")
let kind = try prism.Value.text(listing.kind)
defer kind.release()
let size = try prism.Value.integer((i64)listing.size)
defer size.release()
try tx.value().set(location, "kind", kind)
try tx.value().set(location, "size", size)
journal += location.length + 96
if journal == 0: return true
if not try commit_unless_raced(tx.value()): return false
try committed(database, journal)
return true
## Store one canonical uncompressed loose-object envelope. Content-addressed,
## idempotent only for byte-identical content. Writes id only after success.
## SHA-1 is Git identity, NOT collision-attack detection or release authenticity.
## The bytes go to the object files first, then the repository lists the object.
pub func put_object(database: db.Database*, principal: str, owner: str, name: str, wire: const u8[], id: u8[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
if wire.length > max_object_bytes or id.length != 20:
error(prism.invalid, "invalid repository object size or id output")
let object = try git.decode_object(wire)
var digest: u8[20]
_ = try git.hash_object(object.kind, object.payload, digest)
# Only the owner of an existing repository may add files to the object store.
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
snapshot.value().close()
try object_files.write(try object_files.configured_root(), digest, wire)
var listing = Listing(kind = object.kind, size = wire.length)
for i in 0..<(usize)20: listing.id[i] = digest[i]
try list_objects(database, path, principal, [listing])
for i in 0..<(usize)20: id[i] = digest[i]
pub func get_object(database: db.Database*, principal: str, owner: str, name: str, id: const u8[]) -> interop.Reference[prism.Value]!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
return try checked_object(snapshot.value(), path, id)
## An object this repository lists, read from its file and checked against its id.
## Reads are confined to listed objects: another repository's ids are not reachable.
func checked_object(session: prism.Session*, path: str, id: const u8[]) -> interop.Reference[prism.Value]!:
var object_buffer: u8[256]
let location = try object_path(path, id, object_buffer)
if not try session.contains(location): error(prism.missing, "Git object is not in this repository")
let wire = object_files.read(try object_files.configured_root(), id, max_object_bytes) catch failure:
# A listed object without its file is damage, not absence.
if failure.code == prism.missing: error(prism.invalid, "listed Git object has no file")
error(failure.code, failure.message)
defer wire.release()
return try prism.Value.array(prism.DType.uint8, [(i64)wire.value.length], wire.value)
pub let max_refs: usize = 1024
## One release tag of a repository: its version and the tag object the ref names.
struct ReleaseTag:
var bytes: u8[62]
var length: usize
var parsed: pkg.Version
var target: u8[20]
func version() -> str:
return (str)self.bytes[..<self.length]
func newer(left: pkg.Version, right: pkg.Version) -> bool:
if left.major != right.major: return left.major > right.major
if left.minor != right.minor: return left.minor > right.minor
return left.patch > right.patch
func nibble(byte: u8) -> u8!:
if byte >= '0' and byte <= '9': return byte - '0'
if byte >= 'a' and byte <= 'f': return byte - 'a' + 10
error(prism.invalid, "invalid hex-encoded name")
## The package name `repo_path` stored hex-encoded, decoded into `output`.
func decoded_name(encoded: str, output: u8[]) -> str!:
if encoded.length % 2 != 0 or encoded.length // 2 > output.length: error(prism.invalid, "invalid stored repository name")
for i in 0..<encoded.length // 2:
output[i] = try nibble(encoded.bytes[i * 2]) * 16 + try nibble(encoded.bytes[i * 2 + 1])
return (str)output[..<encoded.length // 2]
## The repository's release tags in `tags`, oldest version first; returns how many.
func release_tags(session: prism.Session*, path: str, tags: ReleaseTag[]) -> usize!:
var refs_buffer: u8[208]
let refs_root = try join([path, "/refs"], refs_buffer)
if not try session.contains(refs_root): return 0
let refs = try session.children(refs_root)
defer refs.release()
var count: usize = 0
for i in 0..<refs.value().count():
let location = try refs.value().at(i)
let label = try session.get(location, "name")
defer label.release()
let ref_name = try label.value().text_at()
let version = release_version(ref_name)
if version.length == 0: continue
if count == tags.length or version.length > tags[count].bytes.length: error(prism.invalid, "too many release tags")
let held = try read_ref(session, location, ref_name)
defer held.release()
var tag = ReleaseTag(length = version.length, parsed = try pkg.parse_version(version))
for j in 0..<version.length: tag.bytes[j] = version.bytes[j]
for j in 0..<(usize)20: tag.target[j] = held.value().bytes()[j]
# Insertion keeps the list sorted: a repository holds at most max_refs tags.
var at = count
while at > 0 and newer(tags[at - 1].parsed, tag.parsed):
tags[at] = tags[at - 1]
at -= 1
tags[at] = tag
count += 1
return count
## `refs/tags/v<major.minor.patch>` names a release; any other tag is an ordinary tag.
func release_version(ref_name: str) -> str:
let prefix = "refs/tags/v"
if not strings.starts_with(ref_name, prefix): return ""
let version = (str)ref_name.bytes[prefix.length..]
_ = pkg.parse_version(version) catch: return ""
return version
pub let max_release_pack: usize = 67108864
## Objects one graph walk or release pack may hold: a release whose tree holds more
## cannot be pushed. 4096 refused luce-browser-engine's tree.
pub let max_graph_objects: usize = 65536
## Open-addressed identity slots of a graph walk, twice its objects; a power of two.
let graph_slots: usize = 131072
## References one graph walk may follow: tree entries, commit trees and tag targets. A
## commit's parents are never followed: history is GitHub's, and the registry needs only
## releases (a tag, its commit and that commit's tree).
let max_graph_edges: usize = 1048576
pub struct TagRelease:
pub var commit: u8[40]
## The commit the release was made from, when its commit names one (`Source-Commit:`):
## `luc publish` sends a snapshot without history, made from a commit on GitHub.
pub var source: u8[40]
pub var has_source: bool
pub var pack_size: usize
pub var definition_size: usize
pub var readme_size: usize
pub var notes_size: usize
pub var tree: u8[20]
## Build the public release of a version tag: a pack holding the tagged commit and its
## complete tree (no history), plus that commit's package.prisma. The definition must
## name this owner, this repository and this version.
## Regular file at a `/`-separated path below `tree`, copied into `output`; 0 when absent.
func tree_file(session: prism.Session*, path: str, tree: const u8[], file: str, output: u8[]) -> usize!:
var current: u8[20]
for i in 0..<(usize)20: current[i] = tree[i]
var start: usize
for end in 0..<file.length + 1:
if end != file.length and file.bytes[end] != '/': continue
let part = (str)file.bytes[start..<end]
let last = end == file.length
start = end + 1
let value = try checked_object(session, path, current)
defer value.release()
let object = try git.decode_object(value.value().bytes())
if object.kind != "tree": return 0
var found = false
var at: usize = 0
while at < object.payload.length:
let entry = try git.decode_tree_entry(object.payload[at..])
at += entry.consumed
if entry.name != part: continue
if last and entry.mode != "100644" and entry.mode != "100755": return 0
if not last and entry.mode != "40000": return 0
for i in 0..<(usize)20: current[i] = entry.id[i]
found = true
break
if not found: return 0
let value = try checked_object(session, path, current)
defer value.release()
let blob = try git.decode_object(value.value().bytes())
if blob.kind != "blob" or blob.payload.length > output.length: return 0
for i in 0..<blob.payload.length: output[i] = blob.payload[i]
return blob.payload.length
## A `Source-Commit: <40 hex digits>` line of a commit message, copied into `source`.
func source_commit(message: const u8[], source: u8[40]*) -> bool:
let key = "Source-Commit: "
var start: usize = 0
for end in 0..<message.length + 1:
if end != message.length and message[end] != '\n': continue
let line = (str)message[start..<end]
start = end + 1
if line.length != key.length + 40 or not strings.starts_with(line, key): continue
var valid = true
for i in 0..<(usize)40:
let digit = line.bytes[key.length + i]
if not ((digit >= '0' and digit <= '9') or (digit >= 'a' and digit <= 'f')): valid = false
if not valid: continue
for i in 0..<(usize)40: (*source)[i] = line.bytes[key.length + i]
return true
return false
func tag_release(database: db.Database*, owner: str, name: str, target: const u8[], version: str,
pack: u8[], definition: u8[], readme: u8[], notes: u8[]) -> TagRelease!:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
var result: TagRelease
# Release notes are mandatory: the tag must be annotated and its message is the notes.
let tag_value = try checked_object(session, path, target)
defer tag_value.release()
let tag_object = try git.decode_object(tag_value.value().bytes())
if tag_object.kind != "tag": error(prism.invalid, "release: release notes are required; tag with: git tag -a v<version> -m \"what changed\"")
let annotated = try git.decode_tag(tag_object.payload)
var first: usize = 0
var last = annotated.message.length
# A signature block is not part of the notes.
let armor = "-----BEGIN"
for i in 0..<annotated.message.length:
if i + armor.length <= annotated.message.length and (str)annotated.message[i..<i + armor.length] == armor and (i == 0 or annotated.message[i - 1] == '\n'):
last = i
break
while first < last and (annotated.message[first] == '\n' or annotated.message[first] == ' ' or annotated.message[first] == '\r' or annotated.message[first] == '\t'): first += 1
while last > first and (annotated.message[last - 1] == '\n' or annotated.message[last - 1] == ' ' or annotated.message[last - 1] == '\r' or annotated.message[last - 1] == '\t'): last -= 1
if last == first: error(prism.invalid, "release: release notes are required; the tag message is empty")
if last - first > notes.length: error(prism.invalid, "release: release notes are too long")
for i in first..<last: notes[i - first] = annotated.message[i]
result.notes_size = last - first
var commit_id: u8[20]
var budget: usize = 1048576
_ = try peel(session, path, target, commit_id, &budget)
let commit_value = try checked_object(session, path, commit_id)
defer commit_value.release()
let commit_object = try git.decode_object(commit_value.value().bytes())
if commit_object.kind != "commit": error(prism.invalid, "release: a release tag must name a commit")
let commit = try git.decode_commit(commit_object.payload)
var tree_id: u8[20]
try git.decode_id(commit.tree, tree_id)
_ = try git.hex_id(commit_id, result.commit)
result.has_source = source_commit(commit.message, &result.source)
for i in 0..<(usize)20: result.tree[i] = tree_id[i]
let root_value = try checked_object(session, path, tree_id)
defer root_value.release()
let root = try git.decode_object(root_value.value().bytes())
var found = false
var at: usize = 0
while at < root.payload.length:
let entry = try git.decode_tree_entry(root.payload[at..])
at += entry.consumed
if entry.name != "package.prisma": continue
if entry.mode != "100644": error(prism.invalid, "release: package.prisma must be a regular file")
let blob_value = try checked_object(session, path, entry.id)
defer blob_value.release()
let blob = try git.decode_object(blob_value.value().bytes())
if blob.payload.length > definition.length: error(prism.invalid, "release: package.prisma is too large")
for i in 0..<blob.payload.length: definition[i] = blob.payload[i]
result.definition_size = blob.payload.length
found = true
if not found: error(prism.invalid, "release: a release needs package.prisma at the repository root")
var declared = try pkg.decode_package(definition[..<result.definition_size])
defer declared.close()
if declared.owner != owner or declared.name != name: error(prism.invalid, "release: package.prisma owner/name must match the repository")
if declared.version != version: error(prism.invalid, "release: package.prisma version must match the release tag")
if declared.readme.length > 0: result.readme_size = try tree_file(session, path, tree_id, declared.readme, readme)
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
for i in 0..<slots.length: slots[i] = 0
var graph = Graph(nodes = nodes, slots = slots)
try graph.add(tree_id, 2)
try graph.validate(session, path)
let objects = try new git.Object[max_graph_objects + 1] --- in memory.heap
defer free(objects) in memory.heap
let leases = try new ObjectLease[max_graph_objects] --- in memory.heap
defer free(leases) in memory.heap
var count: usize = 0
defer release_leases(leases, &count)
objects[0] = commit_object
var total = commit_object.payload.length
for i in 0..<graph.count:
let value = try checked_object(session, path, graph.nodes[i].id)
leases[count] = ObjectLease(value = value)
count += 1
let object = try git.decode_object(value.value().bytes())
if object.payload.length > max_release_pack - total: error(prism.invalid, "release: release source exceeds its size limit")
total += object.payload.length
objects[count] = object
result.pack_size = try git.encode_pack(objects[..<count + 1], pack)
return result
## Write the browsable source of one release: a page per directory and per file.
func source_directory(session: prism.Session*, path: str, tree: const u8[], site_root: str, owner: str, name: str,
coordinate: str, version: str, location: str, html: u8[], depth: usize, budget: usize*) -> !:
if depth > 24: return
let value = try checked_object(session, path, tree)
defer value.release()
let object = try git.decode_object(value.value().bytes())
if object.kind != "tree": return
let listed = try new pages.Listed[1024] --- in memory.heap
defer free(listed) in memory.heap
var count: usize
var at: usize = 0
while at < object.payload.length and count < listed.length:
let entry = try git.decode_tree_entry(object.payload[at..])
at += entry.consumed
if entry.mode == "160000" or entry.mode == "120000": continue
let directory = entry.mode == "40000"
let linked = pages.addressable(entry.name) and location.length + entry.name.length + 1 < 512
var size: usize
if *budget == 0: break
*budget -= 1
var relative: u8[1024]
let inner = try strings.format(relative, f"{location}/{entry.name}") if location.length > 0 else try strings.format(relative, f"{entry.name}")
if directory:
if linked: try source_directory(session, path, entry.id, site_root, owner, name, coordinate, version, inner, html, depth + 1, budget)
else:
let blob_value = try checked_object(session, path, entry.id)
defer blob_value.release()
let blob = try git.decode_object(blob_value.value().bytes())
size = blob.payload.length
# Images are also kept as raw files, so a README can show them.
if linked and pages.image_name(entry.name) and size <= 4194304:
var raw_target: u8[1200]
try site.write_nested(site_root, owner, name, try strings.format(raw_target, f"{version}/raw/{inner}"), blob.payload)
if linked:
var page = pages.page(html)
try pages.blob_page(&page, coordinate, version, inner, blob.payload)
var target: u8[1200]
try site.write_nested(site_root, owner, name, try strings.format(target, f"{version}/blob/{inner}.html"), html[..<page.used])
listed[count] = pages.Listed(name = entry.name, directory = directory, size = size, linked = linked)
count += 1
var page = pages.page(html)
try pages.tree_page(&page, coordinate, version, location, listed[..<count])
var target: u8[1200]
let index = try strings.format(target, f"{version}/tree/{location}/index.html") if location.length > 0 else try strings.format(target, f"{version}/tree/index.html")
try site.write_nested(site_root, owner, name, index, html[..<page.used])
## Regenerate every release's package page and source pages from repository state, so a
## newer registry's rendering reaches releases published by an older one. Done at start;
## a failure for one release is skipped, never fatal.
pub func regenerate_site(database: db.Database*, site_root: str) -> !:
let index_path = try paths.join(site_root, "index")
defer index_path.release()
if not files.exists((c.str)index_path.value): return
let index = try files.read((c.str)index_path.value, limit = 4194304)
defer index.release()
let listing = index.value
var start: usize
for end in 0..<listing.length:
if listing[end] != '\n': continue
let line = (str)listing[start..<end]
start = end + 1
let (coordinate, after) = strings.split_once(line, "\t") else continue
let (latest, _) = strings.split_once(after, "\t") else continue
let (owner, name) = strings.split_once(coordinate, "/") else continue
var relative: u8[300]
let versions_path = try paths.join(site_root, try strings.format(relative, f"{coordinate}/versions"))
defer versions_path.release()
let versions = files.read((c.str)versions_path.value, limit = 4194304) catch: continue
defer versions.release()
var line_start: usize
for line_end in 0..<versions.value.length:
if versions.value[line_end] != '\n': continue
let entry = (str)versions.value[line_start..<line_end]
line_start = line_end + 1
let (version, rest) = strings.split_once(entry, " ") else continue
let (_, after_digest) = strings.split_once(rest, " ") else continue
# the commit, then the source commit when there is one
let (commit_text, _) = strings.split_once(after_digest, " ") else (after_digest, "")
regenerate_release(database, site_root, owner, name, version, strings.trim(commit_text), version == latest) catch:
recover ()
func regenerate_release(database: db.Database*, site_root: str, owner: str, name: str, version: str, commit_text: str, latest: bool) -> !:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
var commit_id: u8[20]
try git.decode_id(commit_text, commit_id)
let value = try checked_object(snapshot.value(), path, commit_id)
defer value.release()
let object = try git.decode_object(value.value().bytes())
let commit = try git.decode_commit(object.payload)
var tree: u8[20]
try git.decode_id(commit.tree, tree)
if latest:
var definition: u8[65536]
let definition_size = try tree_file(snapshot.value(), path, tree, "package.prisma", definition)
if definition_size > 0:
var declared = try pkg.decode_package(definition[..<definition_size])
defer declared.close()
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var readme_size: usize = 0
if declared.readme.length > 0: readme_size = try tree_file(snapshot.value(), path, tree, declared.readme, readme)
try site.write_package_page(site_root, owner, name, definition[..<definition_size], readme[..<readme_size])
try publish_source(database, site_root, owner, name, version, tree)
func publish_source(database: db.Database*, site_root: str, owner: str, name: str, version: str, tree: const u8[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
let html = try new u8[pages.max_page] --- in memory.heap
defer free(html) in memory.heap
var title: u8[160]
let coordinate = try strings.format(title, f"{owner}/{name}")
var budget = max_graph_objects
try source_directory(snapshot.value(), path, tree, site_root, owner, name, coordinate, version, "", html, 0, &budget)
## What `rebuild_site` wrote.
pub struct Rebuilt:
pub var packages: usize
pub var releases: usize
pub var failed: usize
## Repositories `rebuild_site` walks.
let max_repositories: usize = 1024
## Write the public site from repository state alone: every release tag of every
## repository, oldest first, publishes exactly what its push published (release files,
## listing, package page, source browser). The site therefore needs no backup of its
## own. `site_root` should be new: files already there are kept or, for a release file
## with different bytes, refused. A release that fails is reported and skipped.
pub func rebuild_site(database: db.Database*, site_root: str) -> Rebuilt!:
var result: Rebuilt
# Coordinates first, from one snapshot; each release then takes its own.
let coordinates = try new u8[max_repositories * 192] --- in memory.heap
defer free(coordinates) in memory.heap
let lengths = try new usize[max_repositories] in memory.heap
defer free(lengths) in memory.heap
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if try session.contains("/repositories"):
let owners = try session.children("/repositories")
defer owners.release()
for i in 0..<owners.value().count():
let owner_location = try owners.value().at(i)
let owner = paths.base(owner_location)
let repositories_of_owner = try session.children(owner_location)
defer repositories_of_owner.release()
for j in 0..<repositories_of_owner.value().count():
var name_buffer: u8[64]
let name = try decoded_name(paths.base(try repositories_of_owner.value().at(j)), name_buffer)
if result.packages == max_repositories: error(prism.invalid, "too many repositories to rebuild")
let slot = coordinates[result.packages * 192..<result.packages * 192 + 192]
lengths[result.packages] = (try strings.format(slot, f"{owner}/{name}")).length
result.packages += 1
let tags = try new ReleaseTag[max_refs] --- in memory.heap
defer free(tags) in memory.heap
let pack = try new u8[max_release_pack] --- in memory.heap
defer free(pack) in memory.heap
var definition: u8[65536]
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var notes: u8[16384]
for i in 0..<result.packages:
let coordinate = (str)coordinates[i * 192..<i * 192 + lengths[i]]
let (owner, name) = strings.split_once(coordinate, "/") else continue
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var count: usize
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
count = try release_tags(snapshot.value(), path, tags)
for j in 0..<count:
let version = tags[j].version()
rebuild_release(database, site_root, owner, name, version, tags[j].target[0..], pack, definition, readme, notes) catch failure:
print(f"{coordinate} {version}: {failure.message}")
result.failed += 1
recover ()
result.releases += 1
try site.ensure_front_page(site_root)
result.releases -= result.failed
return result
## Publish one release the way its push did.
func rebuild_release(database: db.Database*, site_root: str, owner: str, name: str, version: str, target: const u8[],
pack: u8[], definition: u8[], readme: u8[], notes: u8[]) -> !:
let built = try tag_release(database, owner, name, target, version, pack, definition, readme, notes)
try site.publish_files(site_root, owner, name, version, pack[..<built.pack_size], definition[..<built.definition_size], notes[..<built.notes_size])
try site.publish_listing(site_root, owner, name, version, pack[..<built.pack_size],
(str)built.commit[0..], definition[..<built.definition_size], readme[..<built.readme_size],
(str)built.source[0..] if built.has_source else "")
# As at push time, the source browser never fails a release.
publish_source(database, site_root, owner, name, version, built.tree) catch:
recover ()
## Withdraw one release: delete its tag ref, which a push can never do. Its objects stay
## listed (`trim` drops those no other release reaches); its files leave the site with
## the next `rebuild_site`. The newest release is refused: remove the package instead.
pub func withdraw(database: db.Database*, owner: str, name: str, version: str) -> !:
_ = try pkg.parse_version(version)
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var ref_buffer: u8[96]
let ref_name = try strings.format(ref_buffer, f"refs/tags/v{version}")
var target: u8[20]
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if not try session.contains(path): error(prism.missing, "no such repository")
let tags = try new ReleaseTag[max_refs] --- in memory.heap
defer free(tags) in memory.heap
let count = try release_tags(session, path, tags)
var found = false
for i in 0..<count:
if tags[i].version() != version: continue
if i == count - 1: error(prism.invalid, "the newest release cannot be withdrawn; remove the package instead")
for j in 0..<(usize)20: target[j] = tags[i].target[j]
found = true
if not found: error(prism.missing, "no such release")
var zero: u8[20]
var updates: RefUpdate[1]
updates[0] = RefUpdate(name = ref_name, expected = target[0..], target = zero[0..])
try update_refs(database, owner, owner, name, updates[0..])
func push_capabilities(capabilities: str) -> !:
var start: usize = 0
for i in 0..<capabilities.length + 1:
if i != capabilities.length and capabilities.bytes[i] != ' ': continue
let capability = (str)capabilities.bytes[start..<i]
if capability.length != 0 and capability != "report-status" and capability != "atomic" and capability != "ofs-delta" and capability != "object-format=sha1" and not strings.starts_with(capability, "agent="):
error(prism.invalid, "unsupported push capability")
start = i + 1
struct PackBases:
var session: prism.Session*
var path: str
var value: interop.Reference[prism.Value]?
func close():
if let value = self.value: value.release()
self.value = none
func pack_base(id: const u8[], context: void*?) -> git.Object?!:
let bases = (PackBases*)(context else error(prism.invalid, "missing pack lookup context"))
bases.close()
var buffer: u8[256]
let location = try object_path(bases.path, id, buffer)
if not try bases.session.contains(location): return none
let value = try checked_object(bases.session, bases.path, id)
bases.value = value
return try git.decode_object(value.value().bytes())
## Principal must already be authenticated. This is the receive transaction core,
## not HTTP negotiation or a report-status encoder. A push may only create release tags
## (`refs/tags/v<major.minor.patch>`): branches, other tags and any change to an existing
## ref are refused before anything is stored. Validates a complete or thin pack before
## staging objects, then atomically CAS-publishes graph-checked refs. Failed staging or
## ref publication may leave unreferenced immutable objects, which `trim` removes.
pub func receive_pack(database: db.Database*, principal: str, owner: str, name: str, input: const u8[], site_root: str = "") -> usize!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
let request = try git.decode_push(input)
try push_capabilities(request.capabilities)
if site_root.length == 0: error(prism.invalid, "release: release publication is not configured")
var previous: u8[1280]
var targets: u8[1280]
var updates: RefUpdate[64]
for i in 0..<request.count:
let command = try request.at(i)
if not valid_ref(command.name):
error(prism.invalid, "release: only release tags v<major.minor.patch> can be pushed; branches and other tags belong on GitHub")
try git.decode_id(command.old_id, previous[i * 20..<i * 20 + 20])
try git.decode_id(command.new_id, targets[i * 20..<i * 20 + 20])
updates[i] = RefUpdate(name = command.name, expected = previous[i * 20..<i * 20 + 20], target = targets[i * 20..<i * 20 + 20])
# Released tags never move and are never deleted.
if not zero_id(updates[i].expected) or zero_id(updates[i].target):
error(prism.invalid, "release: a release tag cannot be moved or deleted")
if request.pack.length > 0:
var bases = PackBases(session = snapshot.value(), path = path)
defer bases.close()
var pack = try git.decode_pack_with(request.pack, pack_base, (void*)&bases)
defer pack.close()
bases.close()
snapshot.value().close()
# Validate every packed object's structure, even if no requested ref uses it.
for item in pack.items:
if item.kind == "tree": _ = try git.validate_tree(item.payload)
elif item.kind == "commit": _ = try git.decode_commit(item.payload)
elif item.kind == "tag": _ = try git.decode_tag(item.payload)
# Every object's file is durable before the repository lists any of them.
let listings = try new Listing[pack.items.length] --- in memory.heap
defer free(listings) in memory.heap
let root = try object_files.configured_root()
for i in 0..<pack.items.length:
let item = pack.items[i]
let wire = try new u8[item.payload.length + 32] --- in memory.heap
defer free(wire) in memory.heap
let count = try git.encode_object(item.kind, item.payload, wire)
if count > max_object_bytes: error(prism.invalid, "pushed Git object exceeds limit")
var actual: u8[20]
_ = try git.hash_object(item.kind, item.payload, actual)
if not same(actual, item.id): error(prism.invalid, "stored push object identity mismatch")
try object_files.write(root, actual, wire[..<count])
listings[i] = Listing(kind = item.kind, size = count)
for j in 0..<(usize)20: listings[i].id[j] = actual[j]
try list_objects(database, path, principal, listings)
else:
snapshot.value().close()
# Each tag publishes a release: immutable files are written first, the refs commit,
# then the release is listed.
let pack = try new u8[max_release_pack] --- in memory.heap
defer free(pack) in memory.heap
var definition: u8[65536]
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var notes: u8[16384]
for i in 0..<request.count:
let version = release_version(updates[i].name)
let built = try tag_release(database, owner, name, updates[i].target, version, pack, definition, readme, notes)
try site.publish_files(site_root, owner, name, version, pack[..<built.pack_size], definition[..<built.definition_size], notes[..<built.notes_size])
try update_refs(database, principal, owner, name, updates[..<request.count])
for i in 0..<request.count:
let version = release_version(updates[i].name)
let built = try tag_release(database, owner, name, updates[i].target, version, pack, definition, readme, notes)
try site.publish_listing(site_root, owner, name, version, pack[..<built.pack_size],
(str)built.commit[0..], definition[..<built.definition_size], readme[..<built.readme_size],
(str)built.source[0..] if built.has_source else "")
# The source browser is a convenience: a failure there never fails a release.
publish_source(database, site_root, owner, name, version, built.tree) catch:
recover ()
return request.count
pub let max_ref_name: usize = 1024
pub let max_ref_updates: usize = 64
## Borrowed inputs for one atomic batch. All-zero expected means creation;
## all-zero target means deletion. Both IDs must be exactly 20 bytes.
pub struct RefUpdate:
pub var name: str
pub var expected: const u8[]
pub var target: const u8[]
func zero_id(id: const u8[]) -> bool:
for byte in id:
if byte != 0: return false
return true
## The only refs the registry writes: release tags.
func valid_ref(name: str) -> bool:
return name.length <= max_ref_name and git.valid_ref(name) and release_version(name).length > 0
func ref_path(root: str, name: str, buffer: u8[]) -> str!:
if not valid_ref(name): error(prism.invalid, "invalid repository ref namespace or name")
var encoded: u8[2048]
let digits = b"0123456789abcdef"
for i in 0..<name.length:
encoded[i * 2] = digits[(usize)(name.bytes[i] >> 4)]
encoded[i * 2 + 1] = digits[(usize)(name.bytes[i] & 15)]
return try join([root, "/", (str)encoded[..<name.length * 2]], buffer)
func replaced(name: str, updates: const RefUpdate[]) -> bool:
for update in updates:
if update.name == name: return true
return false
func read_ref(session: prism.Session*, location: str, name: str) -> interop.Reference[prism.Value]!:
let stored_name = try session.get(location, "name")
defer stored_name.release()
if try stored_name.value().text_at() != name: error(prism.invalid, "stored ref name mismatch")
let value = try session.get(location, "id")
errdefer value.release()
if value.value().bytes().length != 20 or zero_id(value.value().bytes()):
error(prism.invalid, "invalid stored ref identity")
return value
## Returns an owning 20-byte ID, or missing. No symbolic refs/HEAD in this API.
pub func get_ref(database: db.Database*, principal: str, owner: str, name: str, ref_name: str) -> interop.Reference[prism.Value]!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
var location_buffer: u8[2304]
let location = try ref_path(root, ref_name, location_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
return try read_ref(snapshot.value(), location, ref_name)
struct GraphNode:
var id: u8[20]
var kind: u8
## A ref target the repository already held: validated, with all it reaches, when
## its own push landed, so a walk stops there.
var trusted: bool
pub let receive_capabilities = "report-status atomic ofs-delta object-format=sha1"
## Follow annotated tags only, within a byte budget.
func peel(session: prism.Session*, path: str, id: const u8[], output: u8[], budget: usize*) -> bool!:
for i in 0..<(usize)20: output[i] = id[i]
var expected: u8 = 0
var peeled = false
for depth in 0..<(usize)65:
let value = try checked_object(session, path, output)
defer value.release()
let wire = value.value().bytes()
if wire.length > *budget: error(prism.invalid, "tag peeling byte limit exceeded")
*budget -= wire.length
let object = try git.decode_object(wire)
let actual = try kind_code(object.kind)
if expected != 0 and expected != actual: error(prism.invalid, "tag target type mismatch")
if object.kind != "tag": return peeled
if depth == 64: error(prism.invalid, "tag peeling depth exceeded")
let tag = try git.decode_tag(object.payload)
expected = try kind_code(tag.kind)
try git.decode_id(tag.object, output)
peeled = true
error(prism.invalid, "invalid tag chain")
## Smart HTTP v0 receive discovery from one authenticated snapshot: the release tags a
## push may build on, oldest version first. Discard output on failure.
pub func advertise_receive(database: db.Database*, principal: str, owner: str, name: str, output: u8[]) -> usize!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
let tags = try new ReleaseTag[max_refs] --- in memory.heap
defer free(tags) in memory.heap
let count = try release_tags(snapshot.value(), path, tags)
var used = try git.encode_packet(git.PacketKind.data, b"# service=git-receive-pack\n", output)
used += try git.encode_packet(git.PacketKind.flush, b"", output[used..])
var line: u8[256]
if count == 0:
let text = try strings.format(line, f"0000000000000000000000000000000000000000 capabilities^{{}}\0{receive_capabilities}\n")
used += try git.encode_packet(git.PacketKind.data, text.bytes, output[used..])
for i in 0..<count:
var encoded: u8[40]
_ = try git.hex_id(tags[i].target, encoded)
# The first line carries the capabilities.
let separator = "\0" if i == 0 else ""
let capabilities = receive_capabilities if i == 0 else ""
let text = try strings.format(line, f"{(str)encoded} refs/tags/v{tags[i].version()}{separator}{capabilities}\n")
used += try git.encode_packet(git.PacketKind.data, text.bytes, output[used..])
used += try git.encode_packet(git.PacketKind.flush, b"", output[used..])
return used
func kind_code(kind: str) -> u8!:
if kind == "blob": return 1
if kind == "tree": return 2
if kind == "commit": return 3
if kind == "tag": return 4
error(prism.invalid, "invalid graph object type")
struct Graph:
var nodes: GraphNode[]
var slots: usize[]
var count: usize
var edges: usize
var probes: usize
func contains(id: const u8[]) -> bool!:
var bucket: usize = 0
for byte in id: bucket = (bucket * 31 + (usize)byte) & (graph_slots - 1)
while true:
self.probes += 1
if self.probes > 16777216: error(prism.invalid, "graph lookup work limit exceeded")
let slot = self.slots[bucket]
if slot == 0: return false
if same(self.nodes[slot - 1].id, id): return true
bucket = (bucket + 1) & (graph_slots - 1)
func add(id: const u8[], kind: u8) -> !:
if id.length != 20 or zero_id(id): error(prism.invalid, "invalid graph identity")
self.edges += 1
if self.edges > max_graph_edges: error(prism.invalid, "graph edge limit exceeded")
var bucket: usize = 0
for byte in id: bucket = (bucket * 31 + (usize)byte) & (graph_slots - 1)
while true:
self.probes += 1
if self.probes > 16777216: error(prism.invalid, "graph lookup work limit exceeded")
let slot = self.slots[bucket]
if slot == 0:
if self.count >= self.nodes.length: error(prism.invalid, "graph object limit exceeded")
self.nodes[self.count] = GraphNode(kind = kind)
for i in 0..<(usize)20: self.nodes[self.count].id[i] = id[i]
self.count += 1
self.slots[bucket] = self.count
return
if same(self.nodes[slot - 1].id, id):
let expected = self.nodes[slot - 1].kind
if kind != 0 and expected != 0 and kind != expected: error(prism.invalid, "graph object type conflict")
if expected == 0: self.nodes[slot - 1].kind = kind
return
bucket = (bucket + 1) & (graph_slots - 1)
## Add `id` as a trusted node: the walk neither reads it nor follows its references.
func trust(id: const u8[]) -> !:
if try self.contains(id): return
try self.add(id, 0)
self.nodes[self.count - 1].trusted = true
func add_hex(id: str, kind: u8) -> !:
var bytes: u8[20]
try git.decode_id(id, bytes)
try self.add(bytes, kind)
func validate(session: prism.Session*, path: str) -> !:
var at: usize = 0
var total: usize = 0
while at < self.count:
if self.nodes[at].trusted:
at += 1
continue
let value = try checked_object(session, path, self.nodes[at].id)
defer value.release()
let bytes = value.value().bytes()
if bytes.length > 268435456 - total: error(prism.invalid, "graph byte limit exceeded")
total += bytes.length
let object = try git.decode_object(bytes)
let actual = try kind_code(object.kind)
if self.nodes[at].kind != 0 and self.nodes[at].kind != actual: error(prism.invalid, "graph target type mismatch")
self.nodes[at].kind = actual
if actual == 2:
_ = try git.validate_tree(object.payload)
var offset: usize = 0
while offset < object.payload.length:
let entry = try git.decode_tree_entry(object.payload[offset..])
# Gitlinks identify commits in another repository, not local edges.
if entry.mode != "160000": try self.add(entry.id, 2 if entry.mode == "40000" else 1)
offset += entry.consumed
elif actual == 3:
# Parents are history, which GitHub keeps: a release is its tag, commit and tree.
try self.add_hex((try git.decode_commit(object.payload)).tree, 2)
elif actual == 4:
let tag = try git.decode_tag(object.payload)
try self.add_hex(tag.object, try kind_code(tag.kind))
at += 1
# a pushed object that names a trusted one says what type it is: that still holds
for i in 0..<self.count:
if not self.nodes[i].trusted or self.nodes[i].kind == 0: continue
let held = checked_object(session, path, self.nodes[i].id) catch failure:
if failure.code == prism.missing: continue
error(failure.code, failure.message)
defer held.release()
if try kind_code(try git.decode_object(held.value().bytes()).kind) != self.nodes[i].kind:
error(prism.invalid, "graph target type mismatch")
## An object a release pack borrows until the pack is encoded.
struct ObjectLease:
var value: interop.Reference[prism.Value]?
func release_leases(leases: ObjectLease[], count: usize*):
for i in 0..<*count:
if let lease = leases[i].value: lease.release()
## Atomic CAS of every release tag in the batch: creation (all-zero expected) for a push,
## deletion (all-zero target) for `withdraw`. Shared guard writes serialize competing
## batches, since Prism detects write overlaps, not read predicates. Graph closure is
## checked in the transaction's snapshot. No identity/date fsck or signature verification.
pub func update_refs(database: db.Database*, principal: str, owner: str, name: str, updates: const RefUpdate[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
if updates.length == 0 or updates.length > max_ref_updates: error(prism.invalid, "invalid ref batch size")
for i in 0..<updates.length:
let update = updates[i]
if not valid_ref(update.name) or update.expected.length != 20 or update.target.length != 20:
error(prism.invalid, "invalid ref update")
if zero_id(update.expected) and zero_id(update.target): error(prism.invalid, "empty ref update")
for j in 0..<i:
if updates[j].name == update.name: error(prism.invalid, "duplicate ref update")
for attempt in 0..<commit_attempts:
if try apply_refs(database, path, principal, updates): return
error(prism.conflict, "ref update kept meeting concurrent writes")
## One attempt of `update_refs`; false when the commit met a concurrent ref batch (the
## guard below makes any two batches of one repository overlap), to retry from scratch.
func apply_refs(database: db.Database*, path: str, principal: str, updates: const RefUpdate[]) -> bool!:
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
let tx = try database.begin()
defer tx.release()
try require_owner(tx.value(), path, principal)
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
for i in 0..<slots.length: slots[i] = 0
var graph = Graph(nodes = nodes, slots = slots)
let children = try tx.value().children(root)
defer children.release()
if children.value().count() > max_refs: error(prism.invalid, "stored ref count exceeds limit")
var retained: usize = 0
for i in 0..<children.value().count():
let child = try children.value().at(i)
let label = try tx.value().get(child, "name")
defer label.release()
let current = try label.value().text_at()
# What the repository's refs reach was checked when they were published (a branch
# a store holds from before releases only, until `trim`, too): the walk below
# covers the pushed objects only.
let held = try read_ref(tx.value(), child, current)
defer held.release()
try graph.trust(held.value().bytes())
if replaced(current, updates): continue
retained += 1
for update in updates:
var location_buffer: u8[2304]
let location = try ref_path(root, update.name, location_buffer)
if try tx.value().contains(location):
let existing = try read_ref(tx.value(), location, update.name)
defer existing.release()
if not same(existing.value().bytes(), update.expected): error(prism.conflict, "stale ref identity")
elif not zero_id(update.expected):
error(prism.conflict, "expected ref is missing")
if not zero_id(update.target):
retained += 1
try graph.add(update.target, 0)
if retained > max_refs: error(prism.invalid, "repository ref limit exceeded")
try graph.validate(tx.value(), path)
if not try tx.value().contains(root): try tx.value().add(root, "directory")
let guard = try prism.Value.boolean(true)
defer guard.release()
try tx.value().set(path, "refs_serialization", guard)
for update in updates:
var location_buffer: u8[2304]
let location = try ref_path(root, update.name, location_buffer)
if zero_id(update.target):
try tx.value().remove(location)
else:
if not try tx.value().contains(location): try tx.value().add(location, "git-ref")
let label = try prism.Value.text(update.name)
defer label.release()
let id = try prism.Value.array(prism.DType.uint8, [20], update.target)
defer id.release()
try tx.value().set(location, "name", label)
try tx.value().set(location, "id", id)
if not try commit_unless_raced(tx.value()): return false
try committed(database, updates.length * (max_ref_name + 64))
return true
## What `remove` took out of the database and the object files.
pub struct Removal:
## Release tags (`refs/tags/v<version>`).
pub var releases: usize
## Objects the repository listed; of those, files deleted and files kept because
## another repository lists the same id.
pub var objects: usize
pub var deleted: usize
pub var shared: usize
## Remove `owner/name` from the database: its record, refs and object listings in one
## transaction (removing the repository element removes everything below it), then the
## object files no other repository lists. Credentials hold no state under the
## repository. The site's files are `site.remove_package`'s job.
##
## Operator-only: `luce-pkg-admin remove`. Online, a push to another repository that
## lists one of these ids in the moment between the check and the deletion would find
## its file gone; run it offline (against the database path) for a guarantee.
pub func remove(database: db.Database*, owner: str, name: str) -> Removal!:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var refs_buffer: u8[208]
let refs_root = try join([path, "/refs"], refs_buffer)
var objects_buffer: u8[208]
let objects_root = try join([path, "/objects"], objects_buffer)
var result: Removal
var ids: u8[]
defer release_ids(&ids)
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if not try session.contains(path): error(prism.missing, "no such repository")
try require_owner(session, path, owner)
if try session.contains(refs_root):
let refs = try session.children(refs_root)
defer refs.release()
for i in 0..<refs.value().count():
let label = try session.get(try refs.value().at(i), "name")
defer label.release()
if release_version(try label.value().text_at()).length > 0: result.releases += 1
let listed = try session.children(objects_root)
defer listed.release()
result.objects = listed.value().count()
if result.objects > 0: ids = try new u8[result.objects * 20] --- in memory.heap
for i in 0..<result.objects:
let location = try listed.value().at(i)
if location.length < 40: error(prism.invalid, "invalid stored object listing")
try git.decode_id((str)location.bytes[location.length - 40..], ids[i * 20..<i * 20 + 20])
var removed = false
for attempt in 0..<commit_attempts:
let tx = try database.begin()
defer tx.release()
if not try tx.value().contains(path): error(prism.missing, "no such repository")
try tx.value().remove(path)
if try commit_unless_raced(tx.value()):
removed = true
break
if not removed: error(prism.conflict, "repository removal kept meeting concurrent writes")
try committed(database, path.length + 256)
let dropped = try drop_unlisted(database, ids, result.objects)
result.deleted = dropped.deleted
result.shared = dropped.shared
return result
## Object files deleted, and kept because some repository still lists their id.
struct Dropped:
var deleted: usize
var shared: usize
## Delete the files of the first `count` ids in `ids` that no repository lists any more.
## Content addressing shares a file among every repository that lists its id; only ids no
## repository lists lose their file.
func drop_unlisted(database: db.Database*, ids: const u8[], count: usize) -> Dropped!:
var result = Dropped(deleted = 0, shared = 0)
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
let others = try new u8[256 * 192] --- in memory.heap
defer free(others) in memory.heap
var other_lengths: usize[256]
var other_count: usize = 0
let owners = try session.children("/repositories")
defer owners.release()
for i in 0..<owners.value().count():
let repositories_of_owner = try session.children(try owners.value().at(i))
defer repositories_of_owner.release()
for j in 0..<repositories_of_owner.value().count():
let other = try repositories_of_owner.value().at(j)
if other_count == other_lengths.length or other.length > 192: error(prism.invalid, "too many repositories to check shared objects")
for k in 0..<other.length: others[other_count * 192 + k] = other.bytes[k]
other_lengths[other_count] = other.length
other_count += 1
let root = try object_files.configured_root()
for i in 0..<count:
let id = ids[i * 20..<i * 20 + 20]
var shared = false
for j in 0..<other_count:
var location_buffer: u8[256]
let location = try object_path((str)others[j * 192..<j * 192 + other_lengths[j]], id, location_buffer)
if try session.contains(location):
shared = true
break
if shared:
result.shared += 1
continue
if try object_files.remove(root, id): result.deleted += 1
return result
## What `trim` took out of one repository.
pub struct Trimmed:
## Release tags kept, and the other refs (branches, ordinary tags) removed.
pub var releases: usize
pub var refs: usize
## Objects the releases reach and stay listed; listings removed; of those, files
## deleted and files kept because another repository lists the same id.
pub var kept: usize
pub var unlisted: usize
pub var deleted: usize
pub var shared: usize
## Object listings removed per transaction.
let trim_batch: usize = 1024
## Reduce `owner/name` to its releases: every ref but the release tags goes, and so does
## every object listing no release reaches through its tag, commit and tree (parents are
## history, which GitHub keeps), then the files of those objects no repository lists.
## The release files on the site are untouched: each was built from exactly what stays.
##
## Operator-only: `luce-pkg-admin trim`. Run it offline (against the database path), as
## `remove`: a push in the meantime could list an object this is about to drop.
pub func trim(database: db.Database*, owner: str, name: str) -> Trimmed!:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var refs_buffer: u8[208]
let refs_root = try join([path, "/refs"], refs_buffer)
var objects_buffer: u8[208]
let objects_root = try join([path, "/objects"], objects_buffer)
var result: Trimmed
var ids: u8[]
defer release_ids(&ids)
# the refs to remove, by location
let doomed = try new u8[max_refs * 2304] --- in memory.heap
defer free(doomed) in memory.heap
var doomed_lengths: usize[1024]
var listed_count: usize = 0
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if not try session.contains(path): error(prism.missing, "no such repository")
try require_owner(session, path, owner)
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
for i in 0..<slots.length: slots[i] = 0
var graph = Graph(nodes = nodes, slots = slots)
if try session.contains(refs_root):
let refs = try session.children(refs_root)
defer refs.release()
for i in 0..<refs.value().count():
let location = try refs.value().at(i)
let label = try session.get(location, "name")
defer label.release()
let ref_name = try label.value().text_at()
if release_version(ref_name).length > 0:
let held = try read_ref(session, location, ref_name)
defer held.release()
try graph.add(held.value().bytes(), 0)
result.releases += 1
else:
if location.length > 2304: error(prism.invalid, "stored ref path too long")
for k in 0..<location.length: doomed[result.refs * 2304 + k] = location.bytes[k]
doomed_lengths[result.refs] = location.length
result.refs += 1
if result.releases == 0: error(prism.invalid, "the repository has no release to keep")
try graph.validate(session, path)
result.kept = graph.count
let listed = try session.children(objects_root)
defer listed.release()
listed_count = listed.value().count()
if listed_count > 0: ids = try new u8[listed_count * 20] --- in memory.heap
for i in 0..<listed_count:
let location = try listed.value().at(i)
if location.length < 40: error(prism.invalid, "invalid stored object listing")
var id: u8[20]
try git.decode_id((str)location.bytes[location.length - 40..], id)
if try graph.contains(id): continue
for k in 0..<(usize)20: ids[result.unlisted * 20 + k] = id[k]
result.unlisted += 1
# the refs first: once they are gone, nothing names the objects about to be unlisted
if result.refs > 0:
var done = false
for attempt in 0..<commit_attempts:
let tx = try database.begin()
defer tx.release()
for i in 0..<result.refs:
let location = (str)doomed[i * 2304..<i * 2304 + doomed_lengths[i]]
if try tx.value().contains(location): try tx.value().remove(location)
if try commit_unless_raced(tx.value()):
done = true
break
if not done: error(prism.conflict, "trimming refs kept meeting concurrent writes")
try committed(database, result.refs * 2304)
var at: usize = 0
while at < result.unlisted:
let end = at + trim_batch if at + trim_batch < result.unlisted else result.unlisted
var done = false
for attempt in 0..<commit_attempts:
let tx = try database.begin()
defer tx.release()
for i in at..<end:
var location_buffer: u8[256]
let location = try object_path(path, ids[i * 20..<i * 20 + 20], location_buffer)
if try tx.value().contains(location): try tx.value().remove(location)
if try commit_unless_raced(tx.value()):
done = true
break
if not done: error(prism.conflict, "trimming object listings kept meeting concurrent writes")
try committed(database, (end - at) * 256)
at = end
let dropped = try drop_unlisted(database, ids, result.unlisted)
result.deleted = dropped.deleted
result.shared = dropped.shared
return result
func release_ids(ids: u8[]*):
if ids.length > 0: free(*ids) in memory.heap