dymokomi/luce-pkg-server
Source of release 0.6.0
0.6.0 / src / repositories.lucb
## Internal repository storage. Principal MUST come from verified authentication,
## never from a request field. Ref updates require bounded graph closure; object
## uploads alone do not validate graph edges or publish refs.
from luce_db import db
import c
from luce_std import files, paths
from luce_prism import prism
import interop
from luce_git import git
import strings
import memory
from luce_pkg import pkg
import time
from luce_std import utf8
import site
import pages
import objects as object_files
pub let max_object_bytes: usize = 67108864
pub let max_pull_requests: usize = 256
pub let max_pull_request_title: usize = 256
pub let max_pull_request_body: usize = 8192
pub let max_pull_request_branch: usize = 255
## A Prism commit is durable on its own: one write-ahead-log transaction. A bake folds
## the log into a new snapshot, in time proportional to the whole store, and holds the
## single writer slot meanwhile. So requests never bake per operation. `committed`
## counts the journal bytes this process wrote since its last bake and bakes once they
## pass `bake_threshold`, well under the journal's 256 MiB cap; the deployment also
## bakes on a timer (`luce-pkg-admin checkpoint <socket>`), covering account writes.
## A writer that meets a running bake queues for up to `commit_wait_ms` instead of
## failing; the HTTP deadlines are minutes.
pub let commit_wait_ms: i64 = 60000
pub let bake_threshold: u64 = 33554432
var unbaked: @u64
## Record `bytes` of committed journal and bake when the unbaked tail passes the
## threshold. One caller claims each bake. A bake that meets another writer or bake is
## left to the next threshold or the timer; the commits are already durable.
func committed(database: db.Database*, bytes: usize) -> !:
let total = unbaked.add((u64)bytes, .relaxed) + (u64)bytes
if total < bake_threshold: return
if unbaked.swap(0, .acq_rel) < bake_threshold: return
_ = database.bake(wait_ms = commit_wait_ms) catch failure:
if failure.code == prism.busy or failure.code == prism.timed_out:
recover -1
error(failure.code, failure.message)
## Attempts at a transaction whose commit met a concurrent write. The transaction is
## rebuilt from a fresh snapshot each time, so every check it makes is repeated.
let commit_attempts: usize = 8
## Commit `tx`, answering false when a concurrent commit overlapped its writes.
func commit_unless_raced(tx: prism.Session*) -> bool!:
_ = tx.commit(wait_ms = commit_wait_ms) catch failure:
if failure.code == prism.conflict: return false
error(failure.code, failure.message)
return true
pub enum PullRequestState as u8:
unknown = 0
open = 1
closed = 2
merged = 3
## A bounded copy of a persisted review record. Branches are short names (without
## refs/heads/); commit IDs are the immutable tips captured at creation/merge.
pub struct PullRequest:
pub var number: i64
pub var author: u8[32]
pub var author_length: usize
pub var title: u8[256]
pub var title_length: usize
pub var body: u8[8192]
pub var body_length: usize
pub var base: u8[255]
pub var base_length: usize
pub var head: u8[255]
pub var head_length: usize
pub var base_id: u8[20]
pub var head_id: u8[20]
pub var merge_id: u8[20]
pub var state: PullRequestState
pub var created_at: i64
pub var updated_at: i64
pub var closed_by: u8[32]
pub var closed_by_length: usize
func valid_owner(value: str) -> bool:
if value.length == 0 or value.length > 32: return false
# Names the public site uses for its own pages and routes.
for reserved in ["applications", "tools", "packages", "assets", "index", "git", "v1", "health"]:
if value == reserved: return false
for byte in value.bytes:
if not ((byte >= 'a' and byte <= 'z') or (byte >= '0' and byte <= '9') or byte == '_'): return false
return true
pub func valid_name(value: str) -> bool:
if value.length == 0 or value.length > 64: return false
for i in 0..<value.length:
let byte = value.bytes[i]
let alnum = (byte >= 'a' and byte <= 'z') or (byte >= '0' and byte <= '9')
if not alnum and (i == 0 or (byte != '_' and byte != '-')): return false
return true
func join(parts: const str[], output: u8[]) -> str!:
var at: usize = 0
for part in parts:
if part.length > output.length - at: error(prism.invalid, "repository path too long")
for byte in part.bytes:
output[at] = byte
at += 1
return (str)output[..<at]
func repo_path(principal: str, owner: str, name: str, output: u8[]) -> str!:
if not valid_owner(owner) or not valid_name(name) or principal != owner:
error(prism.invalid, "invalid repository identity or name")
# Prism names are identifiers, whereas package names may contain hyphens.
# Encode every byte, rather than lossy escaping that could alias two names.
let digits = b"0123456789abcdef"
var encoded_name: u8[128]
for i in 0..<name.length:
encoded_name[i * 2] = digits[(usize)(name.bytes[i] >> 4)]
encoded_name[i * 2 + 1] = digits[(usize)(name.bytes[i] & 15)]
return try join(["/repositories/", owner, "/", (str)encoded_name[..<name.length * 2]], output)
func same(left: const u8[], right: const u8[]) -> bool:
if left.length != right.length: return false
for i in 0..<left.length:
if left[i] != right[i]: return false
return true
func require_owner(session: prism.Session*, path: str, principal: str) -> !:
let value = try session.get(path, "owner")
defer value.release()
if try value.value().text_at() != principal:
error(prism.invalid, "repository owner mismatch")
## Keep Git object bytes in `<database>.objects` (see objects.lucb). Call once at
## startup, before any worker thread, in every process that reads or writes objects.
pub func use_objects(database_path: str) -> !:
try object_files.configure(database_path)
## Invoke once on the database owner before starting concurrent workers.
pub func initialize(database: db.Database*) -> !:
let tx = try database.begin()
defer tx.release()
if try tx.value().contains("/repositories"): return
try tx.value().add("/repositories", "directory")
_ = try tx.value().commit(wait_ms = 1000)
_ = try database.bake(wait_ms = 1000)
## Creates a private repository for an existing account. Duplicate creation is a
## conflict, never replacement. A concurrency conflict must be retried by caller.
pub func create(database: db.Database*, principal: str, name: str) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, principal, name, path_buffer)
var user_buffer: u8[64]
let user = try join(["/users/", principal], user_buffer)
var owner_buffer: u8[64]
let owner_path = try join(["/repositories/", principal], owner_buffer)
var objects_buffer: u8[208]
let objects_path = try join([path, "/objects"], objects_buffer)
let tx = try database.begin()
defer tx.release()
if not try tx.value().contains(user): error(prism.invalid, "repository account does not exist")
if not try tx.value().contains("/repositories"): error(prism.invalid, "repository storage is not initialized")
if try tx.value().contains(path): error(prism.conflict, "repository already exists")
if not try tx.value().contains(owner_path): try tx.value().add(owner_path, "directory")
try tx.value().add(path, "repository")
try tx.value().add(objects_path, "directory")
let owner = try prism.Value.text(principal)
defer owner.release()
try tx.value().set(path, "owner", owner)
let label = try prism.Value.text(name)
defer label.release()
try tx.value().set(path, "name", label)
_ = try tx.value().commit(wait_ms = commit_wait_ms)
try committed(database, path.length * 3 + 256)
func object_path(path: str, id: const u8[], output: u8[]) -> str!:
var encoded_id: u8[40]
_ = try git.hex_id(id, encoded_id)
return try join([path, "/objects/", (str)encoded_id], output)
## What a push stores for one object: its id, kind and envelope size. The bytes are
## already durable in the object files when this is listed in the database.
struct Listing:
var id: u8[20]
var kind: str
var size: usize
## Objects listed per transaction: far below Prism's per-layer operation limit, and
## small enough that a large push commits in steady steps.
let listing_batch: usize = 1024
## List objects whose files are durable. A listing that already exists is left as it is:
## the file store has already checked that its bytes are identical.
func list_objects(database: db.Database*, path: str, principal: str, listings: const Listing[]) -> !:
var at: usize = 0
while at < listings.length:
let end = at + listing_batch if listings.length - at > listing_batch else listings.length
var listed = false
for attempt in 0..<commit_attempts:
if try list_batch(database, path, principal, listings[at..<end]):
listed = true
break
if not listed: error(prism.conflict, "object listing kept meeting concurrent writes")
at = end
## One transaction of `list_objects`; false when its commit met a concurrent write, for
## instance another push listing the same object.
func list_batch(database: db.Database*, path: str, principal: str, listings: const Listing[]) -> bool!:
let tx = try database.begin()
defer tx.release()
try require_owner(tx.value(), path, principal)
var journal: usize = 0
for listing in listings:
var object_buffer: u8[256]
let location = try object_path(path, listing.id, object_buffer)
if try tx.value().contains(location): continue
try tx.value().add(location, "git-object")
let kind = try prism.Value.text(listing.kind)
defer kind.release()
let size = try prism.Value.integer((i64)listing.size)
defer size.release()
try tx.value().set(location, "kind", kind)
try tx.value().set(location, "size", size)
journal += location.length + 96
if journal == 0: return true
if not try commit_unless_raced(tx.value()): return false
try committed(database, journal)
return true
## Store one canonical uncompressed loose-object envelope. Content-addressed,
## idempotent only for byte-identical content. Writes id only after success.
## SHA-1 is Git identity, NOT collision-attack detection or release authenticity.
## The bytes go to the object files first, then the repository lists the object.
pub func put_object(database: db.Database*, principal: str, owner: str, name: str, wire: const u8[], id: u8[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
if wire.length > max_object_bytes or id.length != 20:
error(prism.invalid, "invalid repository object size or id output")
let object = try git.decode_object(wire)
var digest: u8[20]
_ = try git.hash_object(object.kind, object.payload, digest)
# Only the owner of an existing repository may add files to the object store.
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
snapshot.value().close()
try object_files.write(try object_files.configured_root(), digest, wire)
var listing = Listing(kind = object.kind, size = wire.length)
for i in 0..<(usize)20: listing.id[i] = digest[i]
try list_objects(database, path, principal, [listing])
for i in 0..<(usize)20: id[i] = digest[i]
pub func get_object(database: db.Database*, principal: str, owner: str, name: str, id: const u8[]) -> interop.Reference[prism.Value]!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
return try checked_object(snapshot.value(), path, id)
## An object this repository lists, read from its file and checked against its id.
## Reads are confined to listed objects: another repository's ids are not reachable.
func checked_object(session: prism.Session*, path: str, id: const u8[]) -> interop.Reference[prism.Value]!:
var object_buffer: u8[256]
let location = try object_path(path, id, object_buffer)
if not try session.contains(location): error(prism.missing, "Git object is not in this repository")
let wire = object_files.read(try object_files.configured_root(), id, max_object_bytes) catch failure:
# A listed object without its file is damage, not absence.
if failure.code == prism.missing: error(prism.invalid, "listed Git object has no file")
error(failure.code, failure.message)
defer wire.release()
return try prism.Value.array(prism.DType.uint8, [(i64)wire.value.length], wire.value)
pub let max_refs: usize = 1024
## One release tag of a repository: its version and the tag object the ref names.
struct ReleaseTag:
var bytes: u8[62]
var length: usize
var parsed: pkg.Version
var target: u8[20]
func version() -> str:
return (str)self.bytes[..<self.length]
func newer(left: pkg.Version, right: pkg.Version) -> bool:
if left.major != right.major: return left.major > right.major
if left.minor != right.minor: return left.minor > right.minor
return left.patch > right.patch
func nibble(byte: u8) -> u8!:
if byte >= '0' and byte <= '9': return byte - '0'
if byte >= 'a' and byte <= 'f': return byte - 'a' + 10
error(prism.invalid, "invalid hex-encoded name")
## The package name `repo_path` stored hex-encoded, decoded into `output`.
func decoded_name(encoded: str, output: u8[]) -> str!:
if encoded.length % 2 != 0 or encoded.length // 2 > output.length: error(prism.invalid, "invalid stored repository name")
for i in 0..<encoded.length // 2:
output[i] = try nibble(encoded.bytes[i * 2]) * 16 + try nibble(encoded.bytes[i * 2 + 1])
return (str)output[..<encoded.length // 2]
## The repository's release tags in `tags`, oldest version first; returns how many.
func release_tags(session: prism.Session*, path: str, tags: ReleaseTag[]) -> usize!:
var refs_buffer: u8[208]
let refs_root = try join([path, "/refs"], refs_buffer)
if not try session.contains(refs_root): return 0
let refs = try session.children(refs_root)
defer refs.release()
var count: usize = 0
for i in 0..<refs.value().count():
let location = try refs.value().at(i)
let label = try session.get(location, "name")
defer label.release()
let ref_name = try label.value().text_at()
let version = release_version(ref_name)
if version.length == 0: continue
if count == tags.length or version.length > tags[count].bytes.length: error(prism.invalid, "too many release tags")
let held = try read_ref(session, location, ref_name)
defer held.release()
var tag = ReleaseTag(length = version.length, parsed = try pkg.parse_version(version))
for j in 0..<version.length: tag.bytes[j] = version.bytes[j]
for j in 0..<(usize)20: tag.target[j] = held.value().bytes()[j]
# Insertion keeps the list sorted: a repository holds at most max_refs tags.
var at = count
while at > 0 and newer(tags[at - 1].parsed, tag.parsed):
tags[at] = tags[at - 1]
at -= 1
tags[at] = tag
count += 1
return count
## `refs/tags/v<major.minor.patch>` names a release; any other tag is an ordinary tag.
func release_version(ref_name: str) -> str:
let prefix = "refs/tags/v"
if not strings.starts_with(ref_name, prefix): return ""
let version = (str)ref_name.bytes[prefix.length..]
_ = pkg.parse_version(version) catch: return ""
return version
pub let max_release_pack: usize = 67108864
## Objects one graph walk, release pack or fetch may hold: a repository whose tree holds
## more cannot be pushed, released or fetched. 4096 refused luce-browser-engine's tree.
pub let max_graph_objects: usize = 65536
## Open-addressed identity slots of a graph walk, twice its objects; a power of two.
let graph_slots: usize = 131072
## References one graph walk may follow: tree entries, commit trees and parents.
let max_graph_edges: usize = 1048576
pub struct TagRelease:
pub var commit: u8[40]
pub var pack_size: usize
pub var definition_size: usize
pub var readme_size: usize
pub var notes_size: usize
pub var tree: u8[20]
## Build the public release of a version tag: a pack holding the tagged commit and its
## complete tree (no history), plus that commit's package.prisma. The definition must
## name this owner, this repository and this version.
## Regular file at a `/`-separated path below `tree`, copied into `output`; 0 when absent.
func tree_file(session: prism.Session*, path: str, tree: const u8[], file: str, output: u8[]) -> usize!:
var current: u8[20]
for i in 0..<(usize)20: current[i] = tree[i]
var start: usize
for end in 0..<file.length + 1:
if end != file.length and file.bytes[end] != '/': continue
let part = (str)file.bytes[start..<end]
let last = end == file.length
start = end + 1
let value = try checked_object(session, path, current)
defer value.release()
let object = try git.decode_object(value.value().bytes())
if object.kind != "tree": return 0
var found = false
var at: usize = 0
while at < object.payload.length:
let entry = try git.decode_tree_entry(object.payload[at..])
at += entry.consumed
if entry.name != part: continue
if last and entry.mode != "100644" and entry.mode != "100755": return 0
if not last and entry.mode != "40000": return 0
for i in 0..<(usize)20: current[i] = entry.id[i]
found = true
break
if not found: return 0
let value = try checked_object(session, path, current)
defer value.release()
let blob = try git.decode_object(value.value().bytes())
if blob.kind != "blob" or blob.payload.length > output.length: return 0
for i in 0..<blob.payload.length: output[i] = blob.payload[i]
return blob.payload.length
func tag_release(database: db.Database*, owner: str, name: str, target: const u8[], version: str,
pack: u8[], definition: u8[], readme: u8[], notes: u8[]) -> TagRelease!:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
var result: TagRelease
# Release notes are mandatory: the tag must be annotated and its message is the notes.
let tag_value = try checked_object(session, path, target)
defer tag_value.release()
let tag_object = try git.decode_object(tag_value.value().bytes())
if tag_object.kind != "tag": error(prism.invalid, "release: release notes are required; tag with: git tag -a v<version> -m \"what changed\"")
let annotated = try git.decode_tag(tag_object.payload)
var first: usize = 0
var last = annotated.message.length
# A signature block is not part of the notes.
let armor = "-----BEGIN"
for i in 0..<annotated.message.length:
if i + armor.length <= annotated.message.length and (str)annotated.message[i..<i + armor.length] == armor and (i == 0 or annotated.message[i - 1] == '\n'):
last = i
break
while first < last and (annotated.message[first] == '\n' or annotated.message[first] == ' ' or annotated.message[first] == '\r' or annotated.message[first] == '\t'): first += 1
while last > first and (annotated.message[last - 1] == '\n' or annotated.message[last - 1] == ' ' or annotated.message[last - 1] == '\r' or annotated.message[last - 1] == '\t'): last -= 1
if last == first: error(prism.invalid, "release: release notes are required; the tag message is empty")
if last - first > notes.length: error(prism.invalid, "release: release notes are too long")
for i in first..<last: notes[i - first] = annotated.message[i]
result.notes_size = last - first
var commit_id: u8[20]
var budget: usize = 1048576
_ = try peel(session, path, target, commit_id, &budget)
let commit_value = try checked_object(session, path, commit_id)
defer commit_value.release()
let commit_object = try git.decode_object(commit_value.value().bytes())
if commit_object.kind != "commit": error(prism.invalid, "release: a release tag must name a commit")
let commit = try git.decode_commit(commit_object.payload)
var tree_id: u8[20]
try git.decode_id(commit.tree, tree_id)
_ = try git.hex_id(commit_id, result.commit)
for i in 0..<(usize)20: result.tree[i] = tree_id[i]
let root_value = try checked_object(session, path, tree_id)
defer root_value.release()
let root = try git.decode_object(root_value.value().bytes())
var found = false
var at: usize = 0
while at < root.payload.length:
let entry = try git.decode_tree_entry(root.payload[at..])
at += entry.consumed
if entry.name != "package.prisma": continue
if entry.mode != "100644": error(prism.invalid, "release: package.prisma must be a regular file")
let blob_value = try checked_object(session, path, entry.id)
defer blob_value.release()
let blob = try git.decode_object(blob_value.value().bytes())
if blob.payload.length > definition.length: error(prism.invalid, "release: package.prisma is too large")
for i in 0..<blob.payload.length: definition[i] = blob.payload[i]
result.definition_size = blob.payload.length
found = true
if not found: error(prism.invalid, "release: a release needs package.prisma at the repository root")
var declared = try pkg.decode_package(definition[..<result.definition_size])
defer declared.close()
if declared.owner != owner or declared.name != name: error(prism.invalid, "release: package.prisma owner/name must match the repository")
if declared.version != version: error(prism.invalid, "release: package.prisma version must match the release tag")
if declared.readme.length > 0: result.readme_size = try tree_file(session, path, tree_id, declared.readme, readme)
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
for i in 0..<slots.length: slots[i] = 0
var graph = Graph(nodes = nodes, slots = slots)
try graph.add(tree_id, 2)
try graph.validate(session, path)
let objects = try new git.Object[max_graph_objects + 1] --- in memory.heap
defer free(objects) in memory.heap
let leases = try new FetchLease[max_graph_objects] --- in memory.heap
defer free(leases) in memory.heap
var count: usize = 0
defer release_fetch_leases(leases, &count)
objects[0] = commit_object
var total = commit_object.payload.length
for i in 0..<graph.count:
let value = try checked_object(session, path, graph.nodes[i].id)
leases[count] = FetchLease(value = value)
count += 1
let object = try git.decode_object(value.value().bytes())
if object.payload.length > max_release_pack - total: error(prism.invalid, "release: release source exceeds its size limit")
total += object.payload.length
objects[count] = object
result.pack_size = try git.encode_pack(objects[..<count + 1], pack)
return result
## Write the browsable source of one release: a page per directory and per file.
func source_directory(session: prism.Session*, path: str, tree: const u8[], site_root: str, owner: str, name: str,
coordinate: str, version: str, location: str, html: u8[], depth: usize, budget: usize*) -> !:
if depth > 24: return
let value = try checked_object(session, path, tree)
defer value.release()
let object = try git.decode_object(value.value().bytes())
if object.kind != "tree": return
let listed = try new pages.Listed[1024] --- in memory.heap
defer free(listed) in memory.heap
var count: usize
var at: usize = 0
while at < object.payload.length and count < listed.length:
let entry = try git.decode_tree_entry(object.payload[at..])
at += entry.consumed
if entry.mode == "160000" or entry.mode == "120000": continue
let directory = entry.mode == "40000"
let linked = pages.addressable(entry.name) and location.length + entry.name.length + 1 < 512
var size: usize
if *budget == 0: break
*budget -= 1
var relative: u8[1024]
let inner = try strings.format(relative, f"{location}/{entry.name}") if location.length > 0 else try strings.format(relative, f"{entry.name}")
if directory:
if linked: try source_directory(session, path, entry.id, site_root, owner, name, coordinate, version, inner, html, depth + 1, budget)
else:
let blob_value = try checked_object(session, path, entry.id)
defer blob_value.release()
let blob = try git.decode_object(blob_value.value().bytes())
size = blob.payload.length
# Images are also kept as raw files, so a README can show them.
if linked and pages.image_name(entry.name) and size <= 4194304:
var raw_target: u8[1200]
try site.write_nested(site_root, owner, name, try strings.format(raw_target, f"{version}/raw/{inner}"), blob.payload)
if linked:
var page = pages.page(html)
try pages.blob_page(&page, coordinate, version, inner, blob.payload)
var target: u8[1200]
try site.write_nested(site_root, owner, name, try strings.format(target, f"{version}/blob/{inner}.html"), html[..<page.used])
listed[count] = pages.Listed(name = entry.name, directory = directory, size = size, linked = linked)
count += 1
var page = pages.page(html)
try pages.tree_page(&page, coordinate, version, location, listed[..<count])
var target: u8[1200]
let index = try strings.format(target, f"{version}/tree/{location}/index.html") if location.length > 0 else try strings.format(target, f"{version}/tree/index.html")
try site.write_nested(site_root, owner, name, index, html[..<page.used])
## Regenerate every release's package page and source pages from repository state, so a
## newer registry's rendering reaches releases published by an older one. Done at start;
## a failure for one release is skipped, never fatal.
pub func regenerate_site(database: db.Database*, site_root: str) -> !:
let index_path = try paths.join(site_root, "index")
defer index_path.release()
if not files.exists((c.str)index_path.value): return
let index = try files.read((c.str)index_path.value, limit = 4194304)
defer index.release()
let listing = index.value
var start: usize
for end in 0..<listing.length:
if listing[end] != '\n': continue
let line = (str)listing[start..<end]
start = end + 1
let (coordinate, after) = strings.split_once(line, "\t") else continue
let (latest, _) = strings.split_once(after, "\t") else continue
let (owner, name) = strings.split_once(coordinate, "/") else continue
var relative: u8[300]
let versions_path = try paths.join(site_root, try strings.format(relative, f"{coordinate}/versions"))
defer versions_path.release()
let versions = files.read((c.str)versions_path.value, limit = 4194304) catch: continue
defer versions.release()
var line_start: usize
for line_end in 0..<versions.value.length:
if versions.value[line_end] != '\n': continue
let entry = (str)versions.value[line_start..<line_end]
line_start = line_end + 1
let (version, rest) = strings.split_once(entry, " ") else continue
let (_, commit_text) = strings.split_once(rest, " ") else continue
regenerate_release(database, site_root, owner, name, version, strings.trim(commit_text), version == latest) catch:
recover ()
func regenerate_release(database: db.Database*, site_root: str, owner: str, name: str, version: str, commit_text: str, latest: bool) -> !:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
var commit_id: u8[20]
try git.decode_id(commit_text, commit_id)
let value = try checked_object(snapshot.value(), path, commit_id)
defer value.release()
let object = try git.decode_object(value.value().bytes())
let commit = try git.decode_commit(object.payload)
var tree: u8[20]
try git.decode_id(commit.tree, tree)
if latest:
var definition: u8[65536]
let definition_size = try tree_file(snapshot.value(), path, tree, "package.prisma", definition)
if definition_size > 0:
var declared = try pkg.decode_package(definition[..<definition_size])
defer declared.close()
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var readme_size: usize = 0
if declared.readme.length > 0: readme_size = try tree_file(snapshot.value(), path, tree, declared.readme, readme)
try site.write_package_page(site_root, owner, name, definition[..<definition_size], readme[..<readme_size])
try publish_source(database, site_root, owner, name, version, tree)
func publish_source(database: db.Database*, site_root: str, owner: str, name: str, version: str, tree: const u8[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
let html = try new u8[pages.max_page] --- in memory.heap
defer free(html) in memory.heap
var title: u8[160]
let coordinate = try strings.format(title, f"{owner}/{name}")
var budget = max_graph_objects
try source_directory(snapshot.value(), path, tree, site_root, owner, name, coordinate, version, "", html, 0, &budget)
## What `rebuild_site` wrote.
pub struct Rebuilt:
pub var packages: usize
pub var releases: usize
pub var failed: usize
## Repositories `rebuild_site` walks.
let max_repositories: usize = 1024
## Write the public site from repository state alone: every release tag of every
## repository, oldest first, publishes exactly what its push published (release files,
## listing, package page, source browser). The site therefore needs no backup of its
## own. `site_root` should be new: files already there are kept or, for a release file
## with different bytes, refused. A release that fails is reported and skipped.
pub func rebuild_site(database: db.Database*, site_root: str) -> Rebuilt!:
var result: Rebuilt
# Coordinates first, from one snapshot; each release then takes its own.
let coordinates = try new u8[max_repositories * 192] --- in memory.heap
defer free(coordinates) in memory.heap
let lengths = try new usize[max_repositories] in memory.heap
defer free(lengths) in memory.heap
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if try session.contains("/repositories"):
let owners = try session.children("/repositories")
defer owners.release()
for i in 0..<owners.value().count():
let owner_location = try owners.value().at(i)
let owner = paths.base(owner_location)
let repositories_of_owner = try session.children(owner_location)
defer repositories_of_owner.release()
for j in 0..<repositories_of_owner.value().count():
var name_buffer: u8[64]
let name = try decoded_name(paths.base(try repositories_of_owner.value().at(j)), name_buffer)
if result.packages == max_repositories: error(prism.invalid, "too many repositories to rebuild")
let slot = coordinates[result.packages * 192..<result.packages * 192 + 192]
lengths[result.packages] = (try strings.format(slot, f"{owner}/{name}")).length
result.packages += 1
let tags = try new ReleaseTag[max_refs] --- in memory.heap
defer free(tags) in memory.heap
let pack = try new u8[max_release_pack] --- in memory.heap
defer free(pack) in memory.heap
var definition: u8[65536]
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var notes: u8[16384]
for i in 0..<result.packages:
let coordinate = (str)coordinates[i * 192..<i * 192 + lengths[i]]
let (owner, name) = strings.split_once(coordinate, "/") else continue
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var count: usize
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
count = try release_tags(snapshot.value(), path, tags)
for j in 0..<count:
let version = tags[j].version()
rebuild_release(database, site_root, owner, name, version, tags[j].target[0..], pack, definition, readme, notes) catch failure:
print(f"{coordinate} {version}: {failure.message}")
result.failed += 1
recover ()
result.releases += 1
try site.ensure_front_page(site_root)
result.releases -= result.failed
return result
## Publish one release the way its push did.
func rebuild_release(database: db.Database*, site_root: str, owner: str, name: str, version: str, target: const u8[],
pack: u8[], definition: u8[], readme: u8[], notes: u8[]) -> !:
let built = try tag_release(database, owner, name, target, version, pack, definition, readme, notes)
try site.publish_files(site_root, owner, name, version, pack[..<built.pack_size], definition[..<built.definition_size], notes[..<built.notes_size])
try site.publish_listing(site_root, owner, name, version, pack[..<built.pack_size],
(str)built.commit[0..], definition[..<built.definition_size], readme[..<built.readme_size])
# As at push time, the source browser never fails a release.
publish_source(database, site_root, owner, name, version, built.tree) catch:
recover ()
## Withdraw one release: delete its tag ref, which a push can never do. Its objects stay
## listed (the branch history still reaches them); its files leave the site with the
## next `rebuild_site`. The newest release is refused: remove the package instead.
pub func withdraw(database: db.Database*, owner: str, name: str, version: str) -> !:
_ = try pkg.parse_version(version)
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var ref_buffer: u8[96]
let ref_name = try strings.format(ref_buffer, f"refs/tags/v{version}")
var target: u8[20]
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if not try session.contains(path): error(prism.missing, "no such repository")
let tags = try new ReleaseTag[max_refs] --- in memory.heap
defer free(tags) in memory.heap
let count = try release_tags(session, path, tags)
var found = false
for i in 0..<count:
if tags[i].version() != version: continue
if i == count - 1: error(prism.invalid, "the newest release cannot be withdrawn; remove the package instead")
for j in 0..<(usize)20: target[j] = tags[i].target[j]
found = true
if not found: error(prism.missing, "no such release")
var zero: u8[20]
var updates: RefUpdate[1]
updates[0] = RefUpdate(name = ref_name, expected = target[0..], target = zero[0..])
try update_refs(database, owner, owner, name, updates[0..])
func push_capabilities(capabilities: str) -> !:
var start: usize = 0
for i in 0..<capabilities.length + 1:
if i != capabilities.length and capabilities.bytes[i] != ' ': continue
let capability = (str)capabilities.bytes[start..<i]
if capability.length != 0 and capability != "report-status" and capability != "atomic" and capability != "ofs-delta" and capability != "delete-refs" and capability != "object-format=sha1" and not strings.starts_with(capability, "agent="):
error(prism.invalid, "unsupported push capability")
start = i + 1
struct PackBases:
var session: prism.Session*
var path: str
var value: interop.Reference[prism.Value]?
func close():
if let value = self.value: value.release()
self.value = none
func pack_base(id: const u8[], context: void*?) -> git.Object?!:
let bases = (PackBases*)(context else error(prism.invalid, "missing pack lookup context"))
bases.close()
var buffer: u8[256]
let location = try object_path(bases.path, id, buffer)
if not try bases.session.contains(location): return none
let value = try checked_object(bases.session, bases.path, id)
bases.value = value
return try git.decode_object(value.value().bytes())
## Principal must already be authenticated. This is the receive transaction core,
## not HTTP negotiation or a report-status encoder. Validates a complete or thin
## pack before staging objects, then atomically CAS-publishes graph-checked refs.
## Failed staging/ref publication may leave unreferenced immutable objects; retry
## and quota/GC policy belong to the caller. No fast-forward/signature policy yet.
pub func receive_pack(database: db.Database*, principal: str, owner: str, name: str, input: const u8[], site_root: str = "") -> usize!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
let request = try git.decode_push(input)
try push_capabilities(request.capabilities)
var previous: u8[1280]
var targets: u8[1280]
var updates: RefUpdate[64]
for i in 0..<request.count:
let command = try request.at(i)
if not valid_ref(command.name): error(prism.invalid, "unsupported push reference")
try git.decode_id(command.old_id, previous[i * 20..<i * 20 + 20])
try git.decode_id(command.new_id, targets[i * 20..<i * 20 + 20])
updates[i] = RefUpdate(name = command.name, expected = previous[i * 20..<i * 20 + 20], target = targets[i * 20..<i * 20 + 20])
if request.pack.length > 0:
var bases = PackBases(session = snapshot.value(), path = path)
defer bases.close()
var pack = try git.decode_pack_with(request.pack, pack_base, (void*)&bases)
defer pack.close()
bases.close()
snapshot.value().close()
# Validate every packed object's structure, even if no requested ref uses it.
for item in pack.items:
if item.kind == "tree": _ = try git.validate_tree(item.payload)
elif item.kind == "commit": _ = try git.decode_commit(item.payload)
elif item.kind == "tag": _ = try git.decode_tag(item.payload)
# Every object's file is durable before the repository lists any of them.
let listings = try new Listing[pack.items.length] --- in memory.heap
defer free(listings) in memory.heap
let root = try object_files.configured_root()
for i in 0..<pack.items.length:
let item = pack.items[i]
let wire = try new u8[item.payload.length + 32] --- in memory.heap
defer free(wire) in memory.heap
let count = try git.encode_object(item.kind, item.payload, wire)
if count > max_object_bytes: error(prism.invalid, "pushed Git object exceeds limit")
var actual: u8[20]
_ = try git.hash_object(item.kind, item.payload, actual)
if not same(actual, item.id): error(prism.invalid, "stored push object identity mismatch")
try object_files.write(root, actual, wire[..<count])
listings[i] = Listing(kind = item.kind, size = count)
for j in 0..<(usize)20: listings[i].id[j] = actual[j]
try list_objects(database, path, principal, listings)
else:
snapshot.value().close()
# A version tag publishes a release. Released tags never move and are never deleted.
# Immutable files are written first, the refs commit, then the release is listed.
var releases: usize = 0
for i in 0..<request.count:
if release_version(updates[i].name).length == 0: continue
if not zero_id(updates[i].expected): error(prism.invalid, "release: a release tag cannot be moved or deleted")
if site_root.length == 0: error(prism.invalid, "release: release publication is not configured")
releases += 1
if releases == 0:
try update_refs(database, principal, owner, name, updates[..<request.count])
return request.count
let pack = try new u8[max_release_pack] --- in memory.heap
defer free(pack) in memory.heap
var definition: u8[65536]
let readme = try new u8[pages.max_readme] --- in memory.heap
defer free(readme) in memory.heap
var notes: u8[16384]
for i in 0..<request.count:
let version = release_version(updates[i].name)
if version.length == 0: continue
let built = try tag_release(database, owner, name, updates[i].target, version, pack, definition, readme, notes)
try site.publish_files(site_root, owner, name, version, pack[..<built.pack_size], definition[..<built.definition_size], notes[..<built.notes_size])
try update_refs(database, principal, owner, name, updates[..<request.count])
for i in 0..<request.count:
let version = release_version(updates[i].name)
if version.length == 0: continue
let built = try tag_release(database, owner, name, updates[i].target, version, pack, definition, readme, notes)
try site.publish_listing(site_root, owner, name, version, pack[..<built.pack_size],
(str)built.commit[0..], definition[..<built.definition_size], readme[..<built.readme_size])
# The source browser is a convenience: a failure there never fails a release.
publish_source(database, site_root, owner, name, version, built.tree) catch:
recover ()
return request.count
pub let max_ref_name: usize = 1024
pub let max_ref_updates: usize = 64
## Borrowed inputs for one atomic batch. All-zero expected means creation;
## all-zero target means deletion. Both IDs must be exactly 20 bytes.
pub struct RefUpdate:
pub var name: str
pub var expected: const u8[]
pub var target: const u8[]
func zero_id(id: const u8[]) -> bool:
for byte in id:
if byte != 0: return false
return true
func valid_ref(name: str) -> bool:
return name.length <= max_ref_name and git.valid_ref(name) and (strings.starts_with(name, "refs/heads/") or strings.starts_with(name, "refs/tags/"))
func ref_path(root: str, name: str, buffer: u8[]) -> str!:
if not valid_ref(name): error(prism.invalid, "invalid repository ref namespace or name")
var encoded: u8[2048]
let digits = b"0123456789abcdef"
for i in 0..<name.length:
encoded[i * 2] = digits[(usize)(name.bytes[i] >> 4)]
encoded[i * 2 + 1] = digits[(usize)(name.bytes[i] & 15)]
return try join([root, "/", (str)encoded[..<name.length * 2]], buffer)
func prefix_conflict(left: str, right: str) -> bool:
if left == right: return true
if left.length < right.length:
return strings.starts_with(right, left) and right.bytes[left.length] == '/'
return strings.starts_with(left, right) and left.bytes[right.length] == '/'
func replaced(name: str, updates: const RefUpdate[]) -> bool:
for update in updates:
if update.name == name: return true
return false
func read_ref(session: prism.Session*, location: str, name: str) -> interop.Reference[prism.Value]!:
let stored_name = try session.get(location, "name")
defer stored_name.release()
if try stored_name.value().text_at() != name: error(prism.invalid, "stored ref name mismatch")
let value = try session.get(location, "id")
errdefer value.release()
if value.value().bytes().length != 20 or zero_id(value.value().bytes()):
error(prism.invalid, "invalid stored ref identity")
return value
## Returns an owning 20-byte ID, or missing. No symbolic refs/HEAD in this API.
pub func get_ref(database: db.Database*, principal: str, owner: str, name: str, ref_name: str) -> interop.Reference[prism.Value]!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
var location_buffer: u8[2304]
let location = try ref_path(root, ref_name, location_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
return try read_ref(snapshot.value(), location, ref_name)
struct GraphNode:
var id: u8[20]
var kind: u8
## A ref target the repository already held: validated, with all it reaches, when
## its own push landed, so a walk stops there.
var trusted: bool
pub let receive_capabilities = "report-status delete-refs atomic ofs-delta object-format=sha1"
pub let fetch_capabilities = "ofs-delta no-progress object-format=sha1"
## Follow annotated tags only, with a shared discovery/authorization byte budget.
func peel(session: prism.Session*, path: str, id: const u8[], output: u8[], budget: usize*) -> bool!:
for i in 0..<(usize)20: output[i] = id[i]
var expected: u8 = 0
var peeled = false
for depth in 0..<(usize)65:
let value = try checked_object(session, path, output)
defer value.release()
let wire = value.value().bytes()
if wire.length > *budget: error(prism.invalid, "tag peeling byte limit exceeded")
*budget -= wire.length
let object = try git.decode_object(wire)
let actual = try kind_code(object.kind)
if expected != 0 and expected != actual: error(prism.invalid, "tag target type mismatch")
if object.kind != "tag": return peeled
if depth == 64: error(prism.invalid, "tag peeling depth exceeded")
let tag = try git.decode_tag(object.payload)
expected = try kind_code(tag.kind)
try git.decode_id(tag.object, output)
peeled = true
error(prism.invalid, "invalid tag chain")
struct AdvertisementRef:
var name: u8[1024]
var length: usize
var id: u8[20]
func before(left: str, right: str) -> bool:
let count = left.length if left.length < right.length else right.length
for i in 0..<count:
if left.bytes[i] != right.bytes[i]: return left.bytes[i] < right.bytes[i]
return left.length < right.length
## Smart HTTP v0 discovery from one authenticated snapshot. Discard output on
## failure. Upload discovery includes peeled tags and fixed main-branch HEAD.
pub func advertise_receive(database: db.Database*, principal: str, owner: str, name: str, output: u8[], upload: bool = false) -> usize!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
try require_owner(snapshot.value(), path, principal)
let children = try snapshot.value().children(root)
defer children.release()
let count = children.value().count()
if count > max_refs: error(prism.invalid, "stored ref count exceeds limit")
let entries = try new AdvertisementRef[1024] --- in memory.heap
defer free(entries) in memory.heap
var order: usize[1024]
for i in 0..<count:
let child = try children.value().at(i)
let label = try snapshot.value().get(child, "name")
defer label.release()
let ref_name = try label.value().text_at()
if not valid_ref(ref_name): error(prism.invalid, "invalid advertised ref")
var expected_buffer: u8[2304]
if try ref_path(root, ref_name, expected_buffer) != child: error(prism.invalid, "stored ref path mismatch")
let value = try read_ref(snapshot.value(), child, ref_name)
defer value.release()
entries[i] = AdvertisementRef(length = ref_name.length)
for j in 0..<ref_name.length: entries[i].name[j] = ref_name.bytes[j]
for j in 0..<(usize)20: entries[i].id[j] = value.value().bytes()[j]
var at = i
while at > 0:
let previous = order[at - 1]
if not before(ref_name, (str)entries[previous].name[..<entries[previous].length]): break
order[at] = previous
at -= 1
order[at] = i
var used = try git.encode_packet(git.PacketKind.data, b"# service=git-upload-pack\n" if upload else b"# service=git-receive-pack\n", output)
used += try git.encode_packet(git.PacketKind.flush, b"", output[used..])
var line: u8[1200]
var has_head = false
if upload:
for i in 0..<count:
let entry = &entries[i]
if (str)entry.name[..<entry.length] != "refs/heads/main": continue
var encoded: u8[40]
_ = try git.hex_id(entry.id, encoded)
let text = try strings.format(line, f"{(str)encoded} HEAD\0{fetch_capabilities} symref=HEAD:refs/heads/main\n")
used += try git.encode_packet(git.PacketKind.data, text.bytes, output[used..])
has_head = true
let offered = fetch_capabilities if upload else receive_capabilities
if count == 0:
let text = try strings.format(line, f"0000000000000000000000000000000000000000 capabilities^{{}}\0{offered}\n")
used += try git.encode_packet(git.PacketKind.data, text.bytes, output[used..])
var peel_budget: usize = 67108864
for i in 0..<count:
let entry = &entries[order[i]]
var encoded: u8[40]
_ = try git.hex_id(entry.id, encoded)
let separator = "\0" if i == 0 and not has_head else ""
let capabilities = offered if i == 0 and not has_head else ""
let text = try strings.format(line, f"{(str)encoded} {(str)entry.name[..<entry.length]}{separator}{capabilities}\n")
used += try git.encode_packet(git.PacketKind.data, text.bytes, output[used..])
if upload and strings.starts_with((str)entry.name[..<entry.length], "refs/tags/"):
var target: u8[20]
if try peel(snapshot.value(), path, entry.id, target, &peel_budget):
_ = try git.hex_id(target, encoded)
let peeled = try strings.format(line, f"{(str)encoded} {(str)entry.name[..<entry.length]}^{{}}\n")
used += try git.encode_packet(git.PacketKind.data, peeled.bytes, output[used..])
used += try git.encode_packet(git.PacketKind.flush, b"", output[used..])
return used
pub func advertise_upload(database: db.Database*, principal: str, owner: str, name: str, output: u8[]) -> usize!:
return try advertise_receive(database, principal, owner, name, output, true)
func kind_code(kind: str) -> u8!:
if kind == "blob": return 1
if kind == "tree": return 2
if kind == "commit": return 3
if kind == "tag": return 4
error(prism.invalid, "invalid graph object type")
struct Graph:
var nodes: GraphNode[]
var slots: usize[]
var count: usize
var edges: usize
var probes: usize
func contains(id: const u8[]) -> bool!:
var bucket: usize = 0
for byte in id: bucket = (bucket * 31 + (usize)byte) & (graph_slots - 1)
while true:
self.probes += 1
if self.probes > 16777216: error(prism.invalid, "graph lookup work limit exceeded")
let slot = self.slots[bucket]
if slot == 0: return false
if same(self.nodes[slot - 1].id, id): return true
bucket = (bucket + 1) & (graph_slots - 1)
func add(id: const u8[], kind: u8) -> !:
if id.length != 20 or zero_id(id): error(prism.invalid, "invalid graph identity")
self.edges += 1
if self.edges > max_graph_edges: error(prism.invalid, "graph edge limit exceeded")
var bucket: usize = 0
for byte in id: bucket = (bucket * 31 + (usize)byte) & (graph_slots - 1)
while true:
self.probes += 1
if self.probes > 16777216: error(prism.invalid, "graph lookup work limit exceeded")
let slot = self.slots[bucket]
if slot == 0:
if self.count >= self.nodes.length: error(prism.invalid, "graph object limit exceeded")
self.nodes[self.count] = GraphNode(kind = kind)
for i in 0..<(usize)20: self.nodes[self.count].id[i] = id[i]
self.count += 1
self.slots[bucket] = self.count
return
if same(self.nodes[slot - 1].id, id):
let expected = self.nodes[slot - 1].kind
if kind != 0 and expected != 0 and kind != expected: error(prism.invalid, "graph object type conflict")
if expected == 0: self.nodes[slot - 1].kind = kind
return
bucket = (bucket + 1) & (graph_slots - 1)
## Add `id` as a trusted node: the walk neither reads it nor follows its references.
func trust(id: const u8[]) -> !:
if try self.contains(id): return
try self.add(id, 0)
self.nodes[self.count - 1].trusted = true
## Trust the history of every trusted node: the commits and tags its parents and tag
## targets reach. Only those small objects are read, never trees or blobs, so a push
## whose merge reaches an old fork point still stops at history already published.
func trust_history(session: prism.Session*, path: str) -> !:
var at: usize = 0
while at < self.count:
let trusted = self.nodes[at].trusted
var id: u8[20]
for i in 0..<(usize)20: id[i] = self.nodes[at].id[i]
at += 1
if not trusted: continue
let value = try checked_object(session, path, id)
defer value.release()
let object = try git.decode_object(value.value().bytes())
if object.kind == "commit":
let commit = try git.decode_commit(object.payload)
for i in 0..<commit.parent_count:
var parent: u8[20]
try git.decode_id(try commit.parent(i), parent)
try self.trust(parent)
elif object.kind == "tag":
var target: u8[20]
try git.decode_id(try git.decode_tag(object.payload).object, target)
try self.trust(target)
func add_hex(id: str, kind: u8) -> !:
var bytes: u8[20]
try git.decode_id(id, bytes)
try self.add(bytes, kind)
func validate(session: prism.Session*, path: str) -> !:
var at: usize = 0
var total: usize = 0
while at < self.count:
if self.nodes[at].trusted:
at += 1
continue
let value = try checked_object(session, path, self.nodes[at].id)
defer value.release()
let bytes = value.value().bytes()
if bytes.length > 268435456 - total: error(prism.invalid, "graph byte limit exceeded")
total += bytes.length
let object = try git.decode_object(bytes)
let actual = try kind_code(object.kind)
if self.nodes[at].kind != 0 and self.nodes[at].kind != actual: error(prism.invalid, "graph target type mismatch")
self.nodes[at].kind = actual
if actual == 2:
_ = try git.validate_tree(object.payload)
var offset: usize = 0
while offset < object.payload.length:
let entry = try git.decode_tree_entry(object.payload[offset..])
# Gitlinks identify commits in another repository, not local edges.
if entry.mode != "160000": try self.add(entry.id, 2 if entry.mode == "40000" else 1)
offset += entry.consumed
elif actual == 3:
let commit = try git.decode_commit(object.payload)
try self.add_hex(commit.tree, 2)
for i in 0..<commit.parent_count: try self.add_hex(try commit.parent(i), 3)
elif actual == 4:
let tag = try git.decode_tag(object.payload)
try self.add_hex(tag.object, try kind_code(tag.kind))
at += 1
# a pushed object that names a trusted one says what type it is: that still holds
for i in 0..<self.count:
if not self.nodes[i].trusted or self.nodes[i].kind == 0: continue
let held = try checked_object(session, path, self.nodes[i].id)
defer held.release()
if try kind_code(try git.decode_object(held.value().bytes()).kind) != self.nodes[i].kind:
error(prism.invalid, "graph target type mismatch")
struct FetchLease:
var value: interop.Reference[prism.Value]?
func release_fetch_leases(leases: FetchLease[], count: usize*):
for i in 0..<*count:
if let lease = leases[i].value: lease.release()
## Native upload-pack core. Discard output on error. Principal is verified by
## transport. Only advertised ref tips/peeled targets; no arbitrary-ID reads.
## No shallow/filter/side-band/multi-ACK negotiation is advertised by this core.
pub func upload_pack(database: db.Database*, principal: str, owner: str, name: str, input: const u8[], output: u8[]) -> usize!:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
try require_owner(session, path, principal)
let request = try git.decode_fetch(input)
var start: usize = 0
for i in 0..<(request.capabilities.length + 1 if request.capabilities.length > 0 else 0):
if i != request.capabilities.length and request.capabilities.bytes[i] != ' ': continue
let token = (str)request.capabilities.bytes[start..<i]
if token.length != 0 and token != "ofs-delta" and token != "no-progress" and token != "object-format=sha1" and not strings.starts_with(token, "agent="):
error(prism.invalid, "unsupported fetch capability")
start = i + 1
if request.wants == 0: return 0
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
let common_nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(common_nodes) in memory.heap
let common_slots = try new usize[graph_slots] --- in memory.heap
defer free(common_slots) in memory.heap
for i in 0..<slots.length:
slots[i] = 0
common_slots[i] = 0
var wanted = Graph(nodes = nodes, slots = slots)
var common = Graph(nodes = common_nodes, slots = common_slots)
var ids: u8[20480]
var authorized: bool[1024]
for i in 0..<request.wants:
try git.decode_id(try request.want(i), ids[i * 20..<i * 20 + 20])
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
let children = try session.children(root)
defer children.release()
if children.value().count() > max_refs: error(prism.invalid, "stored ref count exceeds limit")
var peel_budget: usize = 67108864
for i in 0..<children.value().count():
let child = try children.value().at(i)
let label = try session.get(child, "name")
defer label.release()
let ref_name = try label.value().text_at()
var location_buffer: u8[2304]
if try ref_path(root, ref_name, location_buffer) != child: error(prism.invalid, "stored ref path mismatch")
let ref = try read_ref(session, child, ref_name)
defer ref.release()
for j in 0..<request.wants:
if same(ref.value().bytes(), ids[j * 20..<j * 20 + 20]): authorized[j] = true
if strings.starts_with(ref_name, "refs/tags/"):
var target: u8[20]
if try peel(session, path, ref.value().bytes(), target, &peel_budget):
for j in 0..<request.wants:
if same(target, ids[j * 20..<j * 20 + 20]): authorized[j] = true
for i in 0..<request.wants:
if not authorized[i]: error(prism.invalid, "fetch want was not advertised")
try wanted.add(ids[i * 20..<i * 20 + 20], 0)
try wanted.validate(session, path)
var acknowledged: u8[20]
var have_common = false
for i in 0..<request.haves:
var id: u8[20]
try git.decode_id(try request.have(i), id)
# Only commits in the wanted closure are negotiation bases. Unknown or
# unrelated haves reveal nothing about other stored objects.
if not try wanted.contains(id): continue
let value = try checked_object(session, path, id)
defer value.release()
if (try git.decode_object(value.value().bytes())).kind != "commit": continue
try common.add(id, 3)
for j in 0..<(usize)20: acknowledged[j] = id[j]
have_common = true
var used: usize = 0
if have_common:
var encoded: u8[40]
_ = try git.hex_id(acknowledged, encoded)
var buffer: u8[64]
let line = try strings.format(buffer, f"ACK {(str)encoded}\n")
used = try git.encode_packet(git.PacketKind.data, line.bytes, output)
else:
used = try git.encode_packet(git.PacketKind.data, b"NAK\n", output)
if not request.done: return used
try common.validate(session, path)
let objects = try new git.Object[max_graph_objects] --- in memory.heap
defer free(objects) in memory.heap
let leases = try new FetchLease[max_graph_objects] --- in memory.heap
defer free(leases) in memory.heap
var count: usize = 0
defer release_fetch_leases(leases, &count)
var total: usize = 0
for i in 0..<wanted.count:
if try common.contains(wanted.nodes[i].id): continue
let value = try checked_object(session, path, wanted.nodes[i].id)
leases[count] = FetchLease(value = value)
count += 1
let object = try git.decode_object(value.value().bytes())
if object.payload.length > 67108864 - total: error(prism.invalid, "fetch object byte limit exceeded")
total += object.payload.length
objects[count - 1] = object
used += try git.encode_pack(objects[..<count], output[used..])
return used
## Atomic CAS of every ref in the batch, including cross-ref prefix checks.
## Shared guard writes serialize competing batches, since Prism detects write
## overlaps, not read predicates. Graph closure is checked in the transaction's
## snapshot. No fast-forward policy, identity/date fsck or signature verification.
pub func update_refs(database: db.Database*, principal: str, owner: str, name: str, updates: const RefUpdate[]) -> !:
var path_buffer: u8[192]
let path = try repo_path(principal, owner, name, path_buffer)
if updates.length == 0 or updates.length > max_ref_updates: error(prism.invalid, "invalid ref batch size")
for i in 0..<updates.length:
let update = updates[i]
if not valid_ref(update.name) or update.expected.length != 20 or update.target.length != 20:
error(prism.invalid, "invalid ref update")
if zero_id(update.expected) and zero_id(update.target): error(prism.invalid, "empty ref update")
for j in 0..<i:
if updates[j].name == update.name: error(prism.invalid, "duplicate ref update")
if not zero_id(update.target) and not zero_id(updates[j].target) and prefix_conflict(update.name, updates[j].name):
error(prism.conflict, "ref prefix conflict")
for attempt in 0..<commit_attempts:
if try apply_refs(database, path, principal, updates): return
error(prism.conflict, "ref update kept meeting concurrent writes")
## One attempt of `update_refs`; false when the commit met a concurrent ref batch (the
## guard below makes any two batches of one repository overlap), to retry from scratch.
func apply_refs(database: db.Database*, path: str, principal: str, updates: const RefUpdate[]) -> bool!:
var root_buffer: u8[208]
let root = try join([path, "/refs"], root_buffer)
let tx = try database.begin()
defer tx.release()
try require_owner(tx.value(), path, principal)
let nodes = try new GraphNode[max_graph_objects] --- in memory.heap
defer free(nodes) in memory.heap
let slots = try new usize[graph_slots] --- in memory.heap
defer free(slots) in memory.heap
for i in 0..<slots.length: slots[i] = 0
var graph = Graph(nodes = nodes, slots = slots)
let children = try tx.value().children(root)
defer children.release()
if children.value().count() > max_refs: error(prism.invalid, "stored ref count exceeds limit")
var retained: usize = 0
for i in 0..<children.value().count():
let child = try children.value().at(i)
let label = try tx.value().get(child, "name")
defer label.release()
let current = try label.value().text_at()
if not valid_ref(current): error(prism.invalid, "invalid stored ref name")
# what the repository's refs reach was checked when they were published: the walk
# below covers the pushed objects only, not the whole history again
let held = try read_ref(tx.value(), child, current)
defer held.release()
try graph.trust(held.value().bytes())
if replaced(current, updates): continue
retained += 1
for update in updates:
if not zero_id(update.target) and prefix_conflict(current, update.name): error(prism.conflict, "ref prefix conflict")
try graph.trust_history(tx.value(), path)
for update in updates:
var location_buffer: u8[2304]
let location = try ref_path(root, update.name, location_buffer)
if try tx.value().contains(location):
let existing = try read_ref(tx.value(), location, update.name)
defer existing.release()
if not same(existing.value().bytes(), update.expected): error(prism.conflict, "stale ref identity")
elif not zero_id(update.expected):
error(prism.conflict, "expected ref is missing")
if not zero_id(update.target):
retained += 1
try graph.add(update.target, 3 if strings.starts_with(update.name, "refs/heads/") else 0)
if retained > max_refs: error(prism.invalid, "repository ref limit exceeded")
try graph.validate(tx.value(), path)
if not try tx.value().contains(root): try tx.value().add(root, "directory")
let guard = try prism.Value.boolean(true)
defer guard.release()
try tx.value().set(path, "refs_serialization", guard)
for update in updates:
var location_buffer: u8[2304]
let location = try ref_path(root, update.name, location_buffer)
if zero_id(update.target):
try tx.value().remove(location)
else:
if not try tx.value().contains(location): try tx.value().add(location, "git-ref")
let label = try prism.Value.text(update.name)
defer label.release()
let id = try prism.Value.array(prism.DType.uint8, [20], update.target)
defer id.release()
try tx.value().set(location, "name", label)
try tx.value().set(location, "id", id)
if not try commit_unless_raced(tx.value()): return false
try committed(database, updates.length * (max_ref_name + 64))
return true
## What `remove` took out of the database and the object files.
pub struct Removal:
## Branches and ordinary tags, then release tags (`refs/tags/v<version>`).
pub var refs: usize
pub var releases: usize
## Objects the repository listed; of those, files deleted and files kept because
## another repository lists the same id.
pub var objects: usize
pub var deleted: usize
pub var shared: usize
## Remove `owner/name` from the database: its record, refs and object listings in one
## transaction (removing the repository element removes everything below it), then the
## object files no other repository lists. Pull requests and credentials hold no state
## under the repository. The site's files are `site.remove_package`'s job.
##
## Operator-only: `luce-pkg-admin remove`. Online, a push to another repository that
## lists one of these ids in the moment between the check and the deletion would find
## its file gone; run it offline (against the database path) for a guarantee.
pub func remove(database: db.Database*, owner: str, name: str) -> Removal!:
var path_buffer: u8[192]
let path = try repo_path(owner, owner, name, path_buffer)
var refs_buffer: u8[208]
let refs_root = try join([path, "/refs"], refs_buffer)
var objects_buffer: u8[208]
let objects_root = try join([path, "/objects"], objects_buffer)
var result: Removal
var ids: u8[]
defer release_ids(&ids)
if true:
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
if not try session.contains(path): error(prism.missing, "no such repository")
try require_owner(session, path, owner)
if try session.contains(refs_root):
let refs = try session.children(refs_root)
defer refs.release()
for i in 0..<refs.value().count():
let label = try session.get(try refs.value().at(i), "name")
defer label.release()
if release_version(try label.value().text_at()).length > 0:
result.releases += 1
else:
result.refs += 1
let listed = try session.children(objects_root)
defer listed.release()
result.objects = listed.value().count()
if result.objects > 0: ids = try new u8[result.objects * 20] --- in memory.heap
for i in 0..<result.objects:
let location = try listed.value().at(i)
if location.length < 40: error(prism.invalid, "invalid stored object listing")
try git.decode_id((str)location.bytes[location.length - 40..], ids[i * 20..<i * 20 + 20])
var removed = false
for attempt in 0..<commit_attempts:
let tx = try database.begin()
defer tx.release()
if not try tx.value().contains(path): error(prism.missing, "no such repository")
try tx.value().remove(path)
if try commit_unless_raced(tx.value()):
removed = true
break
if not removed: error(prism.conflict, "repository removal kept meeting concurrent writes")
try committed(database, path.length + 256)
# Content addressing shares a file among every repository that lists its id; only
# ids no remaining repository lists lose their file.
let snapshot = try database.snapshot()
defer snapshot.release()
let session = snapshot.value()
let others = try new u8[256 * 192] --- in memory.heap
defer free(others) in memory.heap
var other_lengths: usize[256]
var other_count: usize = 0
let owners = try session.children("/repositories")
defer owners.release()
for i in 0..<owners.value().count():
let repositories_of_owner = try session.children(try owners.value().at(i))
defer repositories_of_owner.release()
for j in 0..<repositories_of_owner.value().count():
let other = try repositories_of_owner.value().at(j)
if other_count == other_lengths.length or other.length > 192: error(prism.invalid, "too many repositories to check shared objects")
for k in 0..<other.length: others[other_count * 192 + k] = other.bytes[k]
other_lengths[other_count] = other.length
other_count += 1
let root = try object_files.configured_root()
for i in 0..<result.objects:
let id = ids[i * 20..<i * 20 + 20]
var shared = false
for j in 0..<other_count:
var location_buffer: u8[256]
let location = try object_path((str)others[j * 192..<j * 192 + other_lengths[j]], id, location_buffer)
if try session.contains(location):
shared = true
break
if shared:
result.shared += 1
continue
if try object_files.remove(root, id): result.deleted += 1
return result
func release_ids(ids: u8[]*):
if ids.length > 0: free(*ids) in memory.heap