Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-pkg-server

Source of release 0.6.0

0.6.0 / src / site.lucb

## Static public registry files, served directly by the HTTPS proxy:
##   <site>/index                         one line per package: owner/name TAB latest TAB description
##   <site>/<owner>/<name>/versions       one line per release: version SP sha256 SP commit
##   <site>/<owner>/<name>/<version>.pack     Git pack of the tagged commit and its tree
##   <site>/<owner>/<name>/<version>.prisma   that commit's package.prisma
## Release files are immutable. Everything here is derived from repository state.
import c
from luce_std import files
import interop
import memory
from luce_std import paths
import strings
from luce_pkg import pkg
from luce_prism import prism
import pages

let max_listing: usize = 4194304

func same(left: const u8[], right: const u8[]) -> bool:
    if left.length != right.length: return false
    for i in 0..<left.length:
        if left[i] != right[i]: return false
    return true

## Publish bytes once. An identical existing file is accepted; a different one is refused.
func immutable(path: c.str, bytes: const u8[]) -> !:
    if files.exists(path):
        let current = try files.read(path, limit = bytes.length + 1)
        defer current.release()
        if not same(current.value, bytes): error(prism.invalid, "release: this version already exists with different content")
        return
    try files.write_atomic(path, bytes, synchronize = true)

## `<site>/<owner>/<name>`, created with its parents if missing.
func package_directory(site: str, owner: str, name: str) -> interop.Owned[str]!:
    let parent = try paths.join(site, owner)
    defer parent.release()
    let directory = try paths.join(parent.value, name)
    errdefer directory.release()
    try files.ensure_directory(directory.value)
    return directory

func member(directory: str, version: str, suffix: str) -> interop.Owned[str]!:
    var buffer: u8[96]
    return try paths.join(directory, try strings.format(buffer, f"{version}{suffix}"))

## Step one, before the tag ref is committed: the immutable release files.
pub func publish_files(site: str, owner: str, name: str, version: str, pack: const u8[], definition: const u8[], notes: const u8[]) -> !:
    let directory = try package_directory(site, owner, name)
    defer directory.release()
    let pack_path = try member(directory.value, version, ".pack")
    defer pack_path.release()
    try immutable((c.str)pack_path.value, pack)
    let definition_path = try member(directory.value, version, ".prisma")
    defer definition_path.release()
    try immutable((c.str)definition_path.value, definition)
    let notes_path = try member(directory.value, version, ".notes")
    defer notes_path.release()
    try immutable((c.str)notes_path.value, notes)

## Write one generated page below a package's site directory, creating its parents.
## `relative` is built only from names pages.addressable accepted.
pub func write_nested(site: str, owner: str, name: str, relative: str, bytes: const u8[]) -> !:
    let directory = try package_directory(site, owner, name)
    defer directory.release()
    let path = try paths.join(directory.value, relative)
    defer path.release()
    try files.ensure_directory(paths.directory(path.value))
    try files.write_atomic((c.str)path.value, bytes)

## Rebuild each index line's description and kind from the package's latest definition, so
## an index written by an older registry gains what a newer front page shows.
func refresh_index(site: str) -> !:
    let index_path = try paths.join(site, "index")
    defer index_path.release()
    let index = try read_listing((c.str)index_path.value)
    defer index.release()
    let current = index.value
    if current.length == 0: return
    let buffer = try new u8[max_listing] --- in memory.heap
    defer free(buffer) in memory.heap
    var listing = Text(bytes = buffer)
    var start: usize
    for end in 0..<current.length:
        if current[end] != '\n': continue
        let line = (str)current[start..<end]
        start = end + 1
        let (coordinate, rest) = strings.split_once(line, "\t") else continue
        let (latest, _) = strings.split_once(rest, "\t") else continue
        var relative: u8[400]
        let definition_path = try paths.join(site, try strings.format(relative, f"{coordinate}/{latest}.prisma"))
        defer definition_path.release()
        let bytes = try read_listing((c.str)definition_path.value)
        defer bytes.release()
        var declared = pkg.decode_package(bytes.value) catch:
            # Keep a line whose definition cannot be read exactly as it was.
            try listing.put(line)
            try listing.put("\n")
            continue
        defer declared.close()
        try listing.put(coordinate)
        try listing.put("\t")
        try listing.put(latest)
        try listing.put("\t")
        try listing.put(declared.description)
        try listing.put("\t")
        try listing.put(kind_of(&declared))
        try listing.put("\n")
    if same(buffer[..<listing.used], current): return
    try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)

## Rewrite the front page from the current package list. Done at every start, so a new
## release of the registry refreshes its wording without waiting for a package release.
pub func ensure_front_page(site: str) -> !:
    try files.ensure_directory(site)
    try refresh_index(site)
    let index_path = try paths.join(site, "index")
    defer index_path.release()
    let listing = try read_listing((c.str)index_path.value)
    defer listing.release()
    let html = try new u8[pages.max_page] --- in memory.heap
    defer free(html) in memory.heap
    var front = pages.page(html)
    try pages.index_page(&front, listing.value)
    try write_page(site, html[..<front.used])
    # One page per kind, linked from the tabs at the top of every page.
    for view in ["applications", "tools", "packages"]:
        var kind_page = pages.page(html)
        try pages.index_page(&kind_page, listing.value, view)
        let directory_of_view = try paths.join(site, view)
        defer directory_of_view.release()
        try files.ensure_directory(directory_of_view.value)
        try write_page(directory_of_view.value, html[..<kind_page.used])

## The kind a package declares: application, tool or package.
func kind_of(declared: pkg.Definition*) -> str:
    return declared.kind

func render_readme(readme: const u8[], output: u8[]) -> usize!:
    let document = try prism.decode(readme, prism.Codec.markdown)
    defer document.release()
    let encoded = try prism.encode(document, prism.Codec.html)
    defer encoded.release()
    let bytes = encoded.value().bytes()
    if bytes.length > output.length: error(prism.invalid, "rendered README is too large")
    for i in 0..<bytes.length: output[i] = bytes[i]
    return bytes.length

func write_page(directory: str, bytes: const u8[]) -> !:
    let path = try paths.join(directory, "index.html")
    defer path.release()
    try files.write_atomic((c.str)path.value, bytes, synchronize = true)

## A listing file's bytes, empty when it does not exist yet; the caller releases them.
func read_listing(path: c.str) -> interop.Owned[const u8[]]!:
    if not files.exists(path):
        var empty: u8[]
        return interop.Owned[const u8[]](empty)
    return try files.read(path, limit = max_listing)

func newer(left: pkg.Version, right: pkg.Version) -> bool:
    if left.major != right.major: return left.major > right.major
    if left.minor != right.minor: return left.minor > right.minor
    return left.patch > right.patch

struct Text:
    var bytes: u8[]
    var used: usize

    func put(text: str) -> !:
        if self.used + text.length > self.bytes.length: error(prism.invalid, "registry listing exceeds its bound")
        for i in 0..<text.length: self.bytes[self.used + i] = text.bytes[i]
        self.used += text.length

## Step two, after the tag ref is committed: make the release discoverable.
pub func publish_listing(site: str, owner: str, name: str, version: str, pack: const u8[], commit: str,
                         definition: const u8[], readme: const u8[]) -> !:
    var declared = try pkg.decode_package(definition)
    defer declared.close()
    let description = declared.description
    var digest: u8[32]
    _ = try pkg.source_digest(pack, digest)
    var digest_text: u8[64]
    let digits = "0123456789abcdef"
    for i in 0..<(usize)32:
        digest_text[i * 2] = digits.bytes[(usize)(digest[i] >> 4)]
        digest_text[i * 2 + 1] = digits.bytes[(usize)(digest[i] & 15)]

    let directory = try package_directory(site, owner, name)
    defer directory.release()
    let versions_path = try paths.join(directory.value, "versions")
    defer versions_path.release()
    let versions = try read_listing((c.str)versions_path.value)
    defer versions.release()
    let current = versions.value
    let buffer = try new u8[max_listing] --- in memory.heap
    defer free(buffer) in memory.heap
    var output = Text(bytes = buffer)
    var latest = try pkg.parse_version(version)
    var latest_text = version
    var present = false
    var start: usize
    for end in 0..<current.length:
        if current[end] != '\n': continue
        let line = (str)current[start..<end]
        start = end + 1
        var space = line.length
        for i in 0..<line.length:
            if line.bytes[i] == ' ':
                space = i
                break
        let listed = (str)line.bytes[..<space]
        if listed == version: present = true
        let parsed = pkg.parse_version(listed) catch: continue
        if newer(parsed, latest):
            latest = parsed
            latest_text = listed
        try output.put(line)
        try output.put("\n")
    if not present:
        try output.put(version)
        try output.put(" ")
        try output.put((str)digest_text[0..])
        try output.put(" ")
        try output.put(commit)
        try output.put("\n")
        try files.write_atomic((c.str)versions_path.value, buffer[..<output.used], synchronize = true)

    var coordinate_buffer: u8[160]
    let coordinate = try strings.format(coordinate_buffer, f"{owner}/{name}")
    let index_path = try paths.join(site, "index")
    defer index_path.release()
    let index = try read_listing((c.str)index_path.value)
    defer index.release()
    var listing = Text(bytes = buffer)
    start = 0
    for end in 0..<index.value.length:
        if index.value[end] != '\n': continue
        let line = (str)index.value[start..<end]
        start = end + 1
        if line.length > coordinate.length and strings.starts_with(line, coordinate) and line.bytes[coordinate.length] == '\t': continue
        try listing.put(line)
        try listing.put("\n")
    try listing.put(coordinate)
    try listing.put("\t")
    try listing.put(latest_text)
    try listing.put("\t")
    try listing.put(description)
    try listing.put("\t")
    try listing.put(kind_of(&declared))
    try listing.put("\n")
    try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)

    # Pages. The package page describes the latest release, so an older patch release
    # pushed later refreshes only the versions it lists.
    let html = try new u8[pages.max_page] --- in memory.heap
    defer free(html) in memory.heap
    var front = pages.page(html)
    try pages.index_page(&front, buffer[..<listing.used])
    try write_page(site, html[..<front.used])
    # One page per kind, linked from the tabs at the top of every page.
    for view in ["applications", "tools", "packages"]:
        var kind_page = pages.page(html)
        try pages.index_page(&kind_page, buffer[..<listing.used], view)
        let directory_of_view = try paths.join(site, view)
        defer directory_of_view.release()
        try files.ensure_directory(directory_of_view.value)
        try write_page(directory_of_view.value, html[..<kind_page.used])
    if latest_text == version: try write_package_page(site, owner, name, definition, readme)

## The package page describes the latest release; also rebuilt at registry start.
pub func write_package_page(site: str, owner: str, name: str, definition: const u8[], readme: const u8[]) -> !:
    var declared = try pkg.decode_package(definition)
    defer declared.close()
    let directory = try package_directory(site, owner, name)
    defer directory.release()
    let versions_path = try paths.join(directory.value, "versions")
    defer versions_path.release()
    let html = try new u8[pages.max_page] --- in memory.heap
    defer free(html) in memory.heap
    if true:
        let fragment = try new u8[pages.max_page] --- in memory.heap
        defer free(fragment) in memory.heap
        var rendered_size: usize = 0
        if readme.length > 0:
            # A README that does not render is left out; it never blocks a release.
            rendered_size = render_readme(readme, fragment) catch:
                recover 0
        let rendered = (str)fragment[..<rendered_size]
        let listed = try read_listing((c.str)versions_path.value)
        defer listed.release()
        var detail = pages.page(html)
        try pages.package_page(&detail, &declared, listed.value, rendered, directory.value)
        try write_page(directory.value, html[..<detail.used])

## The packages whose latest release depends on `owner/name`, as `owner/name` lines in
## `output`; returns the bytes used. Only the index's packages are considered.
pub func dependents(site: str, owner: str, name: str, output: u8[]) -> usize!:
    let index_path = try paths.join(site, "index")
    defer index_path.release()
    let listing = try read_listing((c.str)index_path.value)
    defer listing.release()
    var text = Text(bytes = output)
    var start: usize
    for end in 0..<listing.value.length:
        if listing.value[end] != '\n': continue
        let line = (str)listing.value[start..<end]
        start = end + 1
        let (coordinate, rest) = strings.split_once(line, "\t") else continue
        let (latest, _) = strings.split_once(rest, "\t") else continue
        var relative: u8[400]
        let definition_path = try paths.join(site, try strings.format(relative, f"{coordinate}/{latest}.prisma"))
        defer definition_path.release()
        let bytes = try read_listing((c.str)definition_path.value)
        defer bytes.release()
        var declared = pkg.decode_package(bytes.value) catch: continue
        defer declared.close()
        if declared.owner == owner and declared.name == name: continue
        for i in 0..<declared.dependency_count:
            let requirement = try declared.dependency_at(i)
            if requirement.name != name: continue
            if requirement.owner.length > 0 and requirement.owner != owner: continue
            try text.put(coordinate)
            try text.put("\n")
            break
    return text.used

## What `remove_package` took off the site.
pub struct SiteRemoval:
    pub var listed: bool
    pub var releases: usize
    pub var directory: bool

## Take `owner/name` off the public site: its index line, then its directory (release
## packs, definitions, notes, `versions`, the package page and the source browser),
## then rewrite the front and kind pages from the shortened index.
pub func remove_package(site: str, owner: str, name: str) -> SiteRemoval!:
    var result: SiteRemoval
    var coordinate_buffer: u8[160]
    let coordinate = try strings.format(coordinate_buffer, f"{owner}/{name}")
    let index_path = try paths.join(site, "index")
    defer index_path.release()
    let index = try read_listing((c.str)index_path.value)
    defer index.release()
    let buffer = try new u8[max_listing] --- in memory.heap
    defer free(buffer) in memory.heap
    var listing = Text(bytes = buffer)
    var start: usize
    for end in 0..<index.value.length:
        if index.value[end] != '\n': continue
        let line = (str)index.value[start..<end]
        start = end + 1
        if line.length > coordinate.length and strings.starts_with(line, coordinate) and line.bytes[coordinate.length] == '\t':
            result.listed = true
            continue
        try listing.put(line)
        try listing.put("\n")
    if result.listed:
        try files.write_atomic((c.str)index_path.value, buffer[..<listing.used], synchronize = true)
    let owner_directory = try paths.join(site, owner)
    defer owner_directory.release()
    let directory = try paths.join(owner_directory.value, name)
    defer directory.release()
    if files.exists((c.str)directory.value):
        let versions_path = try paths.join(directory.value, "versions")
        defer versions_path.release()
        let versions = try read_listing((c.str)versions_path.value)
        defer versions.release()
        for byte in versions.value:
            if byte == '\n': result.releases += 1
        try files.remove_tree((c.str)directory.value)
        result.directory = true
    try ensure_front_page(site)
    return result