Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-pkg-server

Source of release 0.6.0

0.6.0 / tests / large_objects.lucb

import repositories
from luce_db import db
from luce_auth import auth
from luce_prism import prism
from luce_git import git
import strings
from luce_std import files
import memory
import c

func roundtrip(database: db.Database*, payload: const u8[], write: bool) -> !:
    let wire = try new u8[payload.length + 32] --- in memory.heap
    defer free(wire) in memory.heap
    let n = try git.encode_object("blob", payload, wire)
    var expected: u8[20]
    _ = try git.hash_blob(payload, expected)
    if write:
        var id: u8[20]
        try repositories.put_object(database, "alice", "alice", "large", wire[..<n], id)
        for i in 0..<(usize)20: assert(id[i] == expected[i])
        let before = try database.snapshot()
        defer before.release()
        try repositories.put_object(database, "alice", "alice", "large", wire[..<n], id)
        let after = try database.snapshot()
        defer after.release()
        assert(try before.value().revision() == try after.value().revision())
    let loaded = try repositories.get_object(database, "alice", "alice", "large", expected)
    defer loaded.release()
    let actual = loaded.value().bytes()
    assert(actual.length == n)
    for i in 0..<n: assert(actual[i] == wire[i])

func data_cases(database: db.Database*, bootstrap: c.str, write: bool) -> !:
    let actual_source = try files.read(bootstrap, limit = 33554432)
    defer actual_source.release()
    assert(actual_source.value.length > 1048576)
    try roundtrip(database, actual_source.value, write)
    # Cross the IPC/journal 16 MiB frame limit, not merely the former1MiB cap.
    let payload = try new u8[17825792] --- in memory.heap
    defer free(payload) in memory.heap
    for i in 0..<payload.length: payload[i] = (u8)(i % 251)
    try roundtrip(database, payload, write)
    # Both sides of the first chunk split, including a one-byte final chunk.
    try roundtrip(database, payload[..<1048564], write)
    try roundtrip(database, payload[..<1048563], write)

func limits(database: db.Database*) -> !:
    let payload = try new u8[repositories.max_object_bytes - 14] --- in memory.heap
    defer free(payload) in memory.heap
    for i in 0..<payload.length: payload[i] = (u8)(i % 253)
    try roundtrip(database, payload, true)
    let too_large = try new u8[repositories.max_object_bytes + 1] --- in memory.heap
    defer free(too_large) in memory.heap
    var id: u8[20]
    for i in 0..<(usize)20: id[i] = 0x55
    var rejected = false
    repositories.put_object(database, "alice", "alice", "large", too_large, id) catch failure:
        assert(failure.code == prism.invalid)
        rejected = true
    assert(rejected)
    for byte in id: assert(byte == 0x55)

func expect_missing(database: db.Database*, id: const u8[]) -> !:
    let value = repositories.get_object(database, "alice", "alice", "large", id) catch failure:
        assert(failure.code == prism.missing)
        return
    value.release()
    assert(false)

func expect_invalid(database: db.Database*, id: const u8[]) -> !:
    let value = repositories.get_object(database, "alice", "alice", "large", id) catch failure:
        assert(failure.code == prism.invalid)
        return
    value.release()
    assert(false)

var object_root: u8[1024]
var object_root_length: usize

## `<database>.objects/ab/cdef...` for `id`, as an owned C string.
func object_file(id: const u8[]) -> str!:
    var encoded: u8[40]
    _ = try git.hex_id(id, encoded)
    var buffer: u8[1200]
    return try strings.copy(try strings.format(buffer, f"{(str)object_root[..<object_root_length]}/{(str)encoded[..<2]}/{(str)encoded[2..]}"))

func write_raw(id: const u8[], bytes: const u8[]) -> !:
    let file = try object_file(id)
    defer strings.release(file)
    try files.write((c.str)file, bytes)

## Interruptions and damage to the object files. `stage` leaves what a crash between
## the file and its listing would: a stray temporary and a complete but unlisted file.
## `resume`, in a new process, must not see the object until it is pushed again.
func faults(database: db.Database*, stage_only: bool = false, resume: bool = false) -> !:
    let payload = try new u8[1048590] --- in memory.heap
    defer free(payload) in memory.heap
    for i in 0..<payload.length: payload[i] = (u8)(i % 251)
    let wire = try new u8[payload.length + 32] --- in memory.heap
    defer free(wire) in memory.heap
    let n = try git.encode_object("blob", payload, wire)
    var id: u8[20]
    _ = try git.hash_blob(payload, id)
    if not resume:
        var encoded: u8[40]
        _ = try git.hex_id(id, encoded)
        var buffer: u8[1200]
        let fan_out = try strings.copy(try strings.format(buffer, f"{(str)object_root[..<object_root_length]}/{(str)encoded[..<2]}"))
        defer strings.release(fan_out)
        try files.create_directory((c.str)fan_out)
        let stray = try strings.copy(try strings.format(buffer, f"{(str)object_root[..<object_root_length]}/{(str)encoded[..<2]}/.luce-interrupted"))
        defer strings.release(stray)
        try files.write((c.str)stray, wire[..<1000])
        try write_raw(id, wire[..<n])
    # A file the repository does not list is not an object of it.
    try expect_missing(database, id)
    if stage_only: return
    try repositories.put_object(database, "alice", "alice", "large", wire[..<n], id)
    try roundtrip(database, payload, false)
    # Same-length corruption must fail the identity check before returning bytes.
    let corrupt = try new u8[n] --- in memory.heap
    defer free(corrupt) in memory.heap
    for i in 0..<n: corrupt[i] = wire[i]
    corrupt[n - 1] ^= 1
    try write_raw(id, corrupt)
    try expect_invalid(database, id)
    # A truncated or missing file behind a listing is damage, never absence.
    try write_raw(id, wire[..<n - 1])
    try expect_invalid(database, id)
    let file = try object_file(id)
    defer strings.release(file)
    try files.remove_file((c.str)file)
    try expect_invalid(database, id)
    # Pushing the object again restores its file; the listing already exists.
    try repositories.put_object(database, "alice", "alice", "large", wire[..<n], id)
    try roundtrip(database, payload, false)

pub func main(arguments: c.str[]) -> i32!:
    assert(arguments.length == 4)
    let database = try db.Database.open((str)arguments[1], "large-test-token")
    try repositories.use_objects((str)arguments[1])
    let root = try strings.format(object_root, f"{(str)arguments[1]}.objects")
    object_root_length = root.length
    defer database.release()
    let phase = (str)arguments[2]
    if phase == "read":
        try data_cases(database.value(), arguments[3], false)
    elif phase == "resume":
        try faults(database.value(), resume = true)
    else:
        let authority = try auth.Authority.attach(try database.value().store())
        defer authority.release()
        try authority.value().bootstrap("alice", "disposable-large-test-password")
        try repositories.initialize(database.value())
        try repositories.create(database.value(), "alice", "large")
        if phase == "limits":
            try limits(database.value())
        elif phase == "faults":
            try faults(database.value())
        elif phase == "stage":
            try faults(database.value(), stage_only = true)
        elif phase == "ipc":
            try database.value().listen()
            var socket_buffer: u8[512]
            let socket = try strings.format(socket_buffer, f"{(str)arguments[1]}.sock")
            let client = try db.Database.connect(socket, "large-test-token")
            defer client.release()
            try data_cases(client.value(), arguments[3], true)
        else:
            try data_cases(database.value(), arguments[3], true)
    print("PASS repository large objects")
    return 0