dymokomi/luce-pkg-server
Source of release 0.6.1
## Static public registry files, served directly by the HTTPS proxy:
## <site>/index one line per package: owner/name TAB latest TAB description
## <site>/<owner>/<name>/versions one line per release: version SP sha256 SP commit, then
## SP source when the release commit names the commit
## it was made from (`Source-Commit:`, on GitHub)
## <site>/<owner>/<name>/<version>.pack Git pack of the tagged commit and its tree
## <site>/<owner>/<name>/<version>.prisma that commit's package.prisma
## Release files are immutable. Everything here is derived from repository state.
import c
from luce_std import files
import interop
import memory
from luce_std import paths
import strings
from luce_pkg import pkg
from luce_prism import prism
import pages
let max_listing: usize = 4194304
func same(left: const u8[], right: const u8[]) -> bool:
if left.length != right.length: return false
for i in 0..<left.length:
if left[i] != right[i]: return false
return true
## Publish bytes once. An identical existing file is accepted; a different one is refused.
func immutable(path: c.str, bytes: const u8[]) -> !:
if files.exists(path):
let current = try files.read(path, limit = bytes.length + 1)
defer current.release()
if not same(current.value, bytes): error(prism.invalid, "release: this version already exists with different content")
return
try files.write_atomic(path, bytes, synchronize = true)
## `<site>/<owner>/<name>`, created with its parents if missing.
func package_directory(site: str, owner: str, name: str) -> interop.Owned[str]!:
let parent = try paths.join(site, owner)
defer parent.release()
let directory = try paths.join(parent.value, name)
errdefer directory.release()
try files.ensure_directory(directory.value)
return directory
func member(directory: str, version: str, suffix: str) -> interop.Owned[str]!:
var buffer: u8[96]
return try paths.join(directory, try strings.format(buffer, f"{version}{suffix}"))
## Step one, before the tag ref is committed: the immutable release files.
pub func publish_files(site: str, owner: str, name: str, version: str, pack: const u8[], definition: const u8[], notes: const u8[]) -> !:
let directory = try package_directory(site, owner, name)
defer directory.release()
let pack_path = try member(directory.value, version, ".pack")
defer pack_path.release()
try immutable((c.str)pack_path.value, pack)
let definition_path = try member(directory.value, version, ".prisma")
defer definition_path.release()
try immutable((c.str)definition_path.value, definition)
let notes_path = try member(directory.value, version, ".notes")
defer notes_path.release()
try immutable((c.str)notes_path.value, notes)
## Write one generated page below a package's site directory, creating its parents.
## `relative` is built only from names pages.addressable accepted.
pub func write_nested(site: str, owner: str, name: str, relative: str, bytes: const u8[]) -> !:
let directory = try package_directory(site, owner, name)
defer directory.release()
let path = try paths.join(directory.value, relative)
defer path.release()
try files.ensure_directory(paths.directory(path.value))
try files.write_atomic((c.str)path.value, bytes)
## Rebuild each index line's description and kind from the package's latest definition, so
## an index written by an older registry gains what a newer front page shows.
func refresh_index(site: str) -> !:
let index_path = try paths.join(site, "index")
defer index_path.release()
let index = try read_listing((c.str)index_path.value)
defer index.release()
let current = index.value
if current.length == 0: return
let buffer = try new u8[max_listing] --- in memory.heap
defer free(buffer) in memory.heap
var listing = Text(bytes = buffer)
var start: usize
for end in 0..<current.length:
if current[end] != '\n': continue
let line = (str)current[start..<end]
start = end + 1
let (coordinate, rest) = strings.split_once(line, "\t") else continue
let (latest, _) = strings.split_once(rest, "\t") else continue
var relative: u8[400]
let definition_path = try paths.join(site, try strings.format(relative, f"{coordinate}/{latest}.prisma"))
defer definition_path.release()
let bytes = try read_listing((c.str)definition_path.value)
defer bytes.release()
var declared = pkg.decode_package(bytes.value) catch:
# Keep a line whose definition cannot be read exactly as it was.
try listing.put(line)
try listing.put("\n")
continue
defer declared.close()
try listing.put(coordinate)
try listing.put("\t")
try listing.put(latest)
try listing.put("\t")
try listing.put(declared.description)
try listing.put("\t")
try listing.put(kind_of(&declared))
try listing.put("\n")
if same(buffer[..<listing.used], current): return
try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)
## Rewrite the front page from the current package list. Done at every start, so a new
## release of the registry refreshes its wording without waiting for a package release.
pub func ensure_front_page(site: str) -> !:
try files.ensure_directory(site)
try refresh_index(site)
let index_path = try paths.join(site, "index")
defer index_path.release()
let listing = try read_listing((c.str)index_path.value)
defer listing.release()
let html = try new u8[pages.max_page] --- in memory.heap
defer free(html) in memory.heap
var front = pages.page(html)
try pages.index_page(&front, listing.value)
try write_page(site, html[..<front.used])
# One page per kind, linked from the tabs at the top of every page.
for view in ["applications", "tools", "packages"]:
var kind_page = pages.page(html)
try pages.index_page(&kind_page, listing.value, view)
let directory_of_view = try paths.join(site, view)
defer directory_of_view.release()
try files.ensure_directory(directory_of_view.value)
try write_page(directory_of_view.value, html[..<kind_page.used])
## The kind a package declares: application, tool or package.
func kind_of(declared: pkg.Definition*) -> str:
return declared.kind
func render_readme(readme: const u8[], output: u8[]) -> usize!:
let document = try prism.decode(readme, prism.Codec.markdown)
defer document.release()
let encoded = try prism.encode(document, prism.Codec.html)
defer encoded.release()
let bytes = encoded.value().bytes()
if bytes.length > output.length: error(prism.invalid, "rendered README is too large")
for i in 0..<bytes.length: output[i] = bytes[i]
return bytes.length
func write_page(directory: str, bytes: const u8[]) -> !:
let path = try paths.join(directory, "index.html")
defer path.release()
try files.write_atomic((c.str)path.value, bytes, synchronize = true)
## A listing file's bytes, empty when it does not exist yet; the caller releases them.
func read_listing(path: c.str) -> interop.Owned[const u8[]]!:
if not files.exists(path):
var empty: u8[]
return interop.Owned[const u8[]](empty)
return try files.read(path, limit = max_listing)
func newer(left: pkg.Version, right: pkg.Version) -> bool:
if left.major != right.major: return left.major > right.major
if left.minor != right.minor: return left.minor > right.minor
return left.patch > right.patch
struct Text:
var bytes: u8[]
var used: usize
func put(text: str) -> !:
if self.used + text.length > self.bytes.length: error(prism.invalid, "registry listing exceeds its bound")
for i in 0..<text.length: self.bytes[self.used + i] = text.bytes[i]
self.used += text.length
## Step two, after the tag ref is committed: make the release discoverable.
pub func publish_listing(site: str, owner: str, name: str, version: str, pack: const u8[], commit: str,
definition: const u8[], readme: const u8[], source: str = "") -> !:
var declared = try pkg.decode_package(definition)
defer declared.close()
let description = declared.description
var digest: u8[32]
_ = try pkg.source_digest(pack, digest)
var digest_text: u8[64]
let digits = "0123456789abcdef"
for i in 0..<(usize)32:
digest_text[i * 2] = digits.bytes[(usize)(digest[i] >> 4)]
digest_text[i * 2 + 1] = digits.bytes[(usize)(digest[i] & 15)]
let directory = try package_directory(site, owner, name)
defer directory.release()
let versions_path = try paths.join(directory.value, "versions")
defer versions_path.release()
let versions = try read_listing((c.str)versions_path.value)
defer versions.release()
let current = versions.value
let buffer = try new u8[max_listing] --- in memory.heap
defer free(buffer) in memory.heap
var output = Text(bytes = buffer)
var latest = try pkg.parse_version(version)
var latest_text = version
var present = false
var start: usize
for end in 0..<current.length:
if current[end] != '\n': continue
let line = (str)current[start..<end]
start = end + 1
var space = line.length
for i in 0..<line.length:
if line.bytes[i] == ' ':
space = i
break
let listed = (str)line.bytes[..<space]
if listed == version: present = true
let parsed = pkg.parse_version(listed) catch: continue
if newer(parsed, latest):
latest = parsed
latest_text = listed
try output.put(line)
try output.put("\n")
if not present:
try output.put(version)
try output.put(" ")
try output.put((str)digest_text[0..])
try output.put(" ")
try output.put(commit)
if source.length > 0:
try output.put(" ")
try output.put(source)
try output.put("\n")
try files.write_atomic((c.str)versions_path.value, buffer[..<output.used], synchronize = true)
var coordinate_buffer: u8[160]
let coordinate = try strings.format(coordinate_buffer, f"{owner}/{name}")
let index_path = try paths.join(site, "index")
defer index_path.release()
let index = try read_listing((c.str)index_path.value)
defer index.release()
var listing = Text(bytes = buffer)
start = 0
for end in 0..<index.value.length:
if index.value[end] != '\n': continue
let line = (str)index.value[start..<end]
start = end + 1
if line.length > coordinate.length and strings.starts_with(line, coordinate) and line.bytes[coordinate.length] == '\t': continue
try listing.put(line)
try listing.put("\n")
try listing.put(coordinate)
try listing.put("\t")
try listing.put(latest_text)
try listing.put("\t")
try listing.put(description)
try listing.put("\t")
try listing.put(kind_of(&declared))
try listing.put("\n")
try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)
# Pages. The package page describes the latest release, so an older patch release
# pushed later refreshes only the versions it lists.
let html = try new u8[pages.max_page] --- in memory.heap
defer free(html) in memory.heap
var front = pages.page(html)
try pages.index_page(&front, buffer[..<listing.used])
try write_page(site, html[..<front.used])
# One page per kind, linked from the tabs at the top of every page.
for view in ["applications", "tools", "packages"]:
var kind_page = pages.page(html)
try pages.index_page(&kind_page, buffer[..<listing.used], view)
let directory_of_view = try paths.join(site, view)
defer directory_of_view.release()
try files.ensure_directory(directory_of_view.value)
try write_page(directory_of_view.value, html[..<kind_page.used])
if latest_text == version: try write_package_page(site, owner, name, definition, readme)
## The package page describes the latest release; also rebuilt at registry start.
pub func write_package_page(site: str, owner: str, name: str, definition: const u8[], readme: const u8[]) -> !:
var declared = try pkg.decode_package(definition)
defer declared.close()
let directory = try package_directory(site, owner, name)
defer directory.release()
let versions_path = try paths.join(directory.value, "versions")
defer versions_path.release()
let html = try new u8[pages.max_page] --- in memory.heap
defer free(html) in memory.heap
if true:
let fragment = try new u8[pages.max_page] --- in memory.heap
defer free(fragment) in memory.heap
var rendered_size: usize = 0
if readme.length > 0:
# A README that does not render is left out; it never blocks a release.
rendered_size = render_readme(readme, fragment) catch:
recover 0
let rendered = (str)fragment[..<rendered_size]
let listed = try read_listing((c.str)versions_path.value)
defer listed.release()
var detail = pages.page(html)
try pages.package_page(&detail, &declared, listed.value, rendered, directory.value)
try write_page(directory.value, html[..<detail.used])
## The packages whose latest release depends on `owner/name`, as `owner/name` lines in
## `output`; returns the bytes used. Only the index's packages are considered.
pub func dependents(site: str, owner: str, name: str, output: u8[]) -> usize!:
let index_path = try paths.join(site, "index")
defer index_path.release()
let listing = try read_listing((c.str)index_path.value)
defer listing.release()
var text = Text(bytes = output)
var start: usize
for end in 0..<listing.value.length:
if listing.value[end] != '\n': continue
let line = (str)listing.value[start..<end]
start = end + 1
let (coordinate, rest) = strings.split_once(line, "\t") else continue
let (latest, _) = strings.split_once(rest, "\t") else continue
var relative: u8[400]
let definition_path = try paths.join(site, try strings.format(relative, f"{coordinate}/{latest}.prisma"))
defer definition_path.release()
let bytes = try read_listing((c.str)definition_path.value)
defer bytes.release()
var declared = pkg.decode_package(bytes.value) catch: continue
defer declared.close()
if declared.owner == owner and declared.name == name: continue
for i in 0..<declared.dependency_count:
let requirement = try declared.dependency_at(i)
if requirement.name != name: continue
if requirement.owner.length > 0 and requirement.owner != owner: continue
try text.put(coordinate)
try text.put("\n")
break
return text.used
## What `remove_package` took off the site.
pub struct SiteRemoval:
pub var listed: bool
pub var releases: usize
pub var directory: bool
## Take `owner/name` off the public site: its index line, then its directory (release
## packs, definitions, notes, `versions`, the package page and the source browser),
## then rewrite the front and kind pages from the shortened index.
pub func remove_package(site: str, owner: str, name: str) -> SiteRemoval!:
var result: SiteRemoval
var coordinate_buffer: u8[160]
let coordinate = try strings.format(coordinate_buffer, f"{owner}/{name}")
let index_path = try paths.join(site, "index")
defer index_path.release()
let index = try read_listing((c.str)index_path.value)
defer index.release()
let buffer = try new u8[max_listing] --- in memory.heap
defer free(buffer) in memory.heap
var listing = Text(bytes = buffer)
var start: usize
for end in 0..<index.value.length:
if index.value[end] != '\n': continue
let line = (str)index.value[start..<end]
start = end + 1
if line.length > coordinate.length and strings.starts_with(line, coordinate) and line.bytes[coordinate.length] == '\t':
result.listed = true
continue
try listing.put(line)
try listing.put("\n")
if result.listed:
try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)
let owner_directory = try paths.join(site, owner)
defer owner_directory.release()
let directory = try paths.join(owner_directory.value, name)
defer directory.release()
if files.exists((c.str)directory.value):
let versions_path = try paths.join(directory.value, "versions")
defer versions_path.release()
let versions = try read_listing((c.str)versions_path.value)
defer versions.release()
for byte in versions.value:
if byte == '\n': result.releases += 1
try files.remove_tree((c.str)directory.value)
result.directory = true
try ensure_front_page(site)
return result