dymokomi/luced-message
Source of release 0.5.0
## The mail database: how luced-message keeps mail in luce-prism. One durable Store
## per app holds an identity per account, listing its folders, and one per folder,
## holding its messages:
##
## <account> /f<key> folder: path, name, delimiter, flags, order, total, unseen
## <account>.<key> /state uid_validity, uid_next, modseq
## /m<uid> message: uid, flags, date, size, attachment, sender,
## subject, message_id, in_reply_to, preview, and once
## fetched, raw (the RFC 5322 bytes)
## <account>.outbox /o<n> raw: a message waiting to be sent or filed as a draft
##
## A body lives in a prism extent, written once by its commit; snapshots and the
## window's list reads carry only a reference to it. A folder has its own identity so
## that syncing one folder bakes only that folder.
import interop
import time
from luce_prism import prism
from luce_std.collections import Buffer, TextPool, List
from mail_store import Row, Folder
## How long a commit or bake waits for another writer of the same identity.
let wait_ms: i64 = 1000
## The identity of `account`'s folder with store key `key`, onto `out`.
pub func folder_identity(account: str, key: const u8[], out: Buffer*) -> !:
out.clear()
try out.put_text(account)
try out.put_byte('.')
try out.put(key)
## Mount `identity` unless the store has it already.
pub func ensure(store: prism.Store*, identity: str) -> !:
if try store.mounted(identity):
return
store.mount(identity) catch failure:
# Another engine thread may have mounted it between the look and the mount.
if not try store.mounted(identity):
error(failure.code, failure.message)
## One commit's worth of writes to one identity.
pub struct Batch:
var store: prism.Store*
var identity: Buffer
var session: interop.Reference[prism.Session]?
var path: Buffer
## How many writes the commit will carry.
pub var count: usize
## Writes to `identity`, which is mounted when it is new.
pub static func begin(store: prism.Store*, identity: str) -> Batch!:
try ensure(store, identity)
var batch = Batch(store = store)
errdefer batch.close()
try batch.identity.put_text(identity)
batch.session = try store.begin_identity(identity)
return batch
func session_of() -> prism.Session*!:
let held = self.session else error(prism.closed, "the mail batch is closed")
return try held.get()
func message_path(uid: u32) -> str!:
self.path.clear()
try self.path.put(b"/m")
try self.path.put_number((i64)uid)
return self.path.text()
func element(path: str, kind: str) -> !:
let session = try self.session_of()
if not try session.contains(path):
try session.add(path, kind)
func integer(path: str, name: str, value: i64) -> !:
let held = try prism.Value.integer(value)
defer held.release()
try (try self.session_of()).set(path, name, held)
func text(path: str, name: str, value: str) -> !:
let held = try prism.Value.text(value)
defer held.release()
try (try self.session_of()).set(path, name, held)
func boolean(path: str, name: str, value: bool) -> !:
let held = try prism.Value.boolean(value)
defer held.release()
try (try self.session_of()).set(path, name, held)
## Add message `row`, or replace every field of the one with its UID.
pub func put_row(row: const Row*) -> !:
let path = try self.message_path(row.uid)
try self.element(path, "message")
try self.integer(path, "uid", (i64)row.uid)
try self.integer(path, "flags", (i64)row.flags)
try self.integer(path, "date", row.date)
try self.integer(path, "size", (i64)row.size)
try self.boolean(path, "attachment", row.attachment)
try self.text(path, "sender", row.sender)
try self.text(path, "subject", row.subject)
try self.text(path, "message_id", row.message_id)
try self.text(path, "in_reply_to", row.in_reply_to)
try self.text(path, "preview", row.preview)
self.count += 1
pub func set_flags(uid: u32, flags: u16) -> !:
let path = try self.message_path(uid)
try self.integer(path, "flags", (i64)flags)
self.count += 1
## What a fetched body tells the list: its opening and whether it has attachments.
pub func set_body_summary(uid: u32, preview: str, attachment: bool) -> !:
let path = try self.message_path(uid)
if not try (try self.session_of()).contains(path):
return
try self.text(path, "preview", preview)
try self.boolean(path, "attachment", attachment)
self.count += 1
pub func remove(uid: u32) -> !:
let path = try self.message_path(uid)
let session = try self.session_of()
if try session.contains(path):
try session.remove(path)
self.count += 1
## The raw message `data` of `uid`, as the server sent it.
pub func set_raw(uid: u32, data: const u8[]) -> !:
let path = try self.message_path(uid)
let session = try self.session_of()
if not try session.contains(path):
return
try put_raw(session, path, data)
self.count += 1
## The folder's mailbox state.
pub func put_state(uid_validity: u32, uid_next: u32, modseq: u64) -> !:
try self.element("/state", "state")
try self.integer("/state", "uid_validity", (i64)uid_validity)
try self.integer("/state", "uid_next", (i64)uid_next)
try self.integer("/state", "modseq", (i64)modseq)
self.count += 1
## Folder `folder`, listed at place `order`.
pub func put_folder(folder: const Folder*, order: usize) -> !:
self.path.clear()
try self.path.put(b"/f")
try self.path.put_text(folder.key)
let path = self.path.text()
try self.element(path, "folder")
try self.text(path, "path", folder.path)
try self.text(path, "name", folder.name)
try self.integer(path, "delimiter", (i64)folder.delimiter)
try self.integer(path, "flags", (i64)folder.flags)
try self.integer(path, "order", (i64)order)
try self.integer(path, "total", (i64)folder.total)
try self.integer(path, "unseen", (i64)folder.unseen)
self.count += 1
## Remove every element but those named in `keep` (paths, sorted or not).
pub func remove_others(keep: const str[]) -> !:
let session = try self.session_of()
let children = try session.children("/")
defer children.release()
for i in 0..<children.value().count():
let path = try children.value().at(i)
var kept = false
for wanted in keep:
if wanted == path:
kept = true
break
if not kept:
try session.remove(path)
self.count += 1
## Commit the writes so far, if there are any, and go on writing in a new commit:
## a long sync shows its progress commit by commit.
pub func flush() -> !:
if self.count == 0:
return
let session = try self.session_of()
_ = try session.commit(wait_ms = wait_ms)
if let held = self.session:
held.release()
self.session = none
self.count = 0
self.session = try self.store.begin_identity(self.identity.text())
## Commit the writes, if there are any, and end the batch.
pub func commit() -> !:
if self.count > 0:
_ = try (try self.session_of()).commit(wait_ms = wait_ms)
self.close()
pub func close():
if let held = self.session:
held.release()
self.session = none
self.identity.destroy()
self.path.destroy()
func put_raw(session: prism.Session*, path: str, data: const u8[]) -> !:
let shape: i64[1] = [(i64)data.length]
let value = try prism.Value.array(prism.DType.uint8, shape, data)
defer value.release()
try session.set(path, "raw", value)
## Apply the identity's unbaked commits to its tables, so later reads and opens do not
## replay them. The engine bakes a folder after syncing it.
pub func bake(store: prism.Store*, identity: str) -> !:
if try store.mounted(identity):
_ = try store.bake_identity(identity, wait_ms = wait_ms)
## A folder's mailbox state and, for the engine, its messages' UIDs and flags.
pub struct Ledger:
pub var uid_validity: u32
pub var uid_next: u32
pub var modseq: u64
## Sorted by UID.
pub var uids: List[u32]
pub var flags: List[u16]
## Whether each message's raw text is in the store.
pub var bodies: List[bool]
pub func find(uid: u32) -> usize?:
var low: usize = 0
var high = self.uids.length()
while low < high:
let middle = (low + high) // 2
let found = self.uids.at(middle)
if found == uid:
return middle
if found < uid:
low = middle + 1
else:
high = middle
return none
pub func last_uid() -> u32:
return self.uids.at(self.uids.length() - 1) if self.uids.length() > 0 else 0
## How many messages are not marked seen.
pub func unseen() -> u32:
var count: u32 = 0
for bits in self.flags.view():
if (bits & 1) == 0:
count += 1
return count
## Record `uid` with `flags` and whether its body is stored, keeping UIDs sorted.
pub func put(uid: u32, flags: u16, body: bool = false) -> !:
if let at = self.find(uid):
self.flags.set(at, flags)
self.bodies.set(at, body or self.bodies.at(at))
return
try self.uids.append(uid)
try self.flags.append(flags)
try self.bodies.append(body)
var at = self.uids.length() - 1
while at > 0 and self.uids.at(at - 1) > uid:
self.uids.set(at, self.uids.at(at - 1))
self.flags.set(at, self.flags.at(at - 1))
self.bodies.set(at, self.bodies.at(at - 1))
at -= 1
self.uids.set(at, uid)
self.flags.set(at, flags)
self.bodies.set(at, body)
pub func has_body(uid: u32) -> bool:
let at = self.find(uid) else return false
return self.bodies.at(at)
pub func remove_at(at: usize):
self.uids.remove(at)
self.flags.remove(at)
self.bodies.remove(at)
pub func remove(uid: u32):
if let at = self.find(uid):
self.remove_at(at)
pub func clear():
self.uids.clear()
self.flags.clear()
self.bodies.clear()
pub func close():
self.uids.destroy()
self.flags.destroy()
self.bodies.destroy()
func ledger_row(context: void*?, row: const prism.RowView*) -> !:
let ledger = (Ledger*)(context else error(prism.invalid, "no ledger"))
let uid = row.integer("uid") else return
try ledger.put((u32)uid, (u16)(row.integer("flags") else 0), row.has("raw"))
## The ledger of folder identity `identity`: empty when the store has none yet.
pub func read_ledger(store: prism.Store*, identity: str) -> Ledger!:
var ledger: Ledger
errdefer ledger.close()
if not try store.mounted(identity):
return ledger
let snap = try store.snapshot_identity(identity)
defer snap.release()
let session = try snap.get()
if try session.contains("/state"):
ledger.uid_validity = (u32)(try integer_of(session, "/state", "uid_validity"))
ledger.uid_next = (u32)(try integer_of(session, "/state", "uid_next"))
ledger.modseq = (u64)(try integer_of(session, "/state", "modseq"))
try session.each_child("/", ledger_row, &ledger)
return ledger
func integer_of(session: prism.Session*, path: str, name: str) -> i64!:
let value = try session.get(path, name)
defer value.release()
return try value.value().integer_at()
## Rows read from a folder identity, their text kept in `pool`.
pub struct Rows:
pub var rows: List[Row]
pub var pool: TextPool
pub func close():
self.rows.destroy()
self.pool.destroy()
func collect_row(context: void*?, view: const prism.RowView*) -> !:
let into = (Rows*)(context else error(prism.invalid, "no rows"))
let uid = view.integer("uid") else return
try into.rows.append(Row(uid = (u32)uid, flags = (u16)(view.integer("flags") else 0), date = view.integer("date") else 0,
size = (u32)(view.integer("size") else 0), attachment = view.boolean("attachment") else false,
sender = try into.pool.keep_text((view.text("sender") else "").bytes),
subject = try into.pool.keep_text((view.text("subject") else "").bytes),
message_id = try into.pool.keep_text((view.text("message_id") else "").bytes),
in_reply_to = try into.pool.keep_text((view.text("in_reply_to") else "").bytes),
preview = try into.pool.keep_text((view.text("preview") else "").bytes)))
## Every message of folder identity `identity`, in the order they were added.
pub func read_rows(store: prism.Store*, identity: str) -> Rows!:
var out: Rows
errdefer out.close()
if not try store.mounted(identity):
return out
let snap = try store.snapshot_identity(identity)
defer snap.release()
try (try snap.get()).each_child("/", collect_row, &out)
return out
## Folders read from an account identity, their text kept in `pool`.
pub struct Folders:
pub var folders: List[Folder]
var orders: List[i64]
pub var pool: TextPool
pub func close():
self.folders.destroy()
self.orders.destroy()
self.pool.destroy()
func collect_folder(context: void*?, view: const prism.RowView*) -> !:
let into = (Folders*)(context else error(prism.invalid, "no folders"))
let path = view.path()
if path.length < 3:
return
let order = view.integer("order") else 0
let folder = Folder(path = try into.pool.keep_text((view.text("path") else "").bytes),
name = try into.pool.keep_text((view.text("name") else "").bytes),
delimiter = (u8)(view.integer("delimiter") else 0), flags = (u32)(view.integer("flags") else 0),
key = try into.pool.keep_text(path.bytes[2..]), total = (u32)(view.integer("total") else 0),
unseen = (u32)(view.integer("unseen") else 0))
# Keep the listing in the engine's order: INBOX first, then as the server lists.
try into.folders.append(folder)
try into.orders.append(order)
var at = into.folders.length() - 1
while at > 0 and into.orders.at(at - 1) > order:
into.folders.set(at, into.folders.at(at - 1))
into.orders.set(at, into.orders.at(at - 1))
at -= 1
into.folders.set(at, folder)
into.orders.set(at, order)
## The folders of account identity `account`, in listing order.
pub func read_folders(store: prism.Store*, account: str) -> Folders!:
var out: Folders
errdefer out.close()
if not try store.mounted(account):
return out
let snap = try store.snapshot_identity(account)
defer snap.release()
try (try snap.get()).each_child("/", collect_folder, &out)
return out
## Empty account `account`'s identity and every folder identity it lists: Accounts ›
## Remove. The identities stay in the catalog, empty; prism has no unmount.
pub func forget_account(held: interop.Reference[prism.Store], account: str) -> !:
let store = try held.get()
var listed = try read_folders(store, account)
defer listed.close()
var identity: Buffer
defer identity.destroy()
for folder in listed.folders.view():
try folder_identity(account, folder.key.bytes, &identity)
try clear(store, identity.text())
try outbox_identity(account, &identity)
try clear(store, identity.text())
try clear(store, account)
func clear(store: prism.Store*, identity: str) -> !:
if not try store.mounted(identity):
return
var batch = try Batch.begin(store, identity)
defer batch.close()
let none_kept: str[1] = [""]
try batch.remove_others(none_kept[..<0])
try batch.commit()
try bake(store, identity)
# mark: messages ============================================================================
## The raw text of message `uid` of folder identity `identity`, onto `out`; false when
## the body is not in the store yet.
pub func read_raw(store: prism.Store*, identity: str, uid: u32, out: Buffer*) -> bool!:
var path: Buffer
defer path.destroy()
try path.put(b"/m")
try path.put_number((i64)uid)
return try read_raw_at(store, identity, path.text(), out)
func read_raw_at(store: prism.Store*, identity: str, path: str, out: Buffer*) -> bool!:
if not try store.mounted(identity):
return false
let snap = try store.snapshot_identity(identity)
defer snap.release()
let session = try snap.get()
if not try session.has(path, "raw"):
return false
let value = try session.get(path, "raw")
defer value.release()
out.clear()
try out.put(value.value().bytes())
return true
## Whether message `uid` of folder identity `identity` has its body in the store.
pub func has_body(store: prism.Store*, identity: str, uid: u32) -> bool!:
if not try store.mounted(identity):
return false
var path: Buffer
defer path.destroy()
try path.put(b"/m")
try path.put_number((i64)uid)
let snap = try store.snapshot_identity(identity)
defer snap.release()
return try (try snap.get()).has(path.text(), "raw")
## The outbox of `account`: messages queued for its engine to send or file.
pub func outbox_identity(account: str, out: Buffer*) -> !:
out.clear()
try out.put_text(account)
try out.put(b".outbox")
var queued: @u64
## Queue `message` (RFC 5322 text) in `account`'s outbox; its element path, which the
## engine's send and draft commands name.
pub func queue(held: interop.Reference[prism.Store], account: str, message: str) -> interop.Owned[str]!:
let store = try held.get()
var identity: Buffer
defer identity.destroy()
try outbox_identity(account, &identity)
try ensure(store, identity.text())
var path: Buffer
defer path.destroy()
try path.put(b"/o")
try path.put_number((i64)time.now())
# A dot would start a slot in a prism path.
try path.put_byte('-')
try path.put_number((i64)queued.add(1, .relaxed))
let tx = try store.begin_identity(identity.text())
defer tx.release()
let session = try tx.get()
try session.add(path.text(), "outgoing")
try put_raw(session, path.text(), message.bytes)
_ = try session.commit(wait_ms = wait_ms)
return try interop.copy_text(path.text())
## The queued message at `path` in `account`'s outbox, onto `out`; false when gone.
pub func read_queued(store: prism.Store*, account: str, path: str, out: Buffer*) -> bool!:
var identity: Buffer
defer identity.destroy()
try outbox_identity(account, &identity)
return try read_raw_at(store, identity.text(), path, out)
## Drop the queued message at `path` from `account`'s outbox: it was sent or filed.
pub func unqueue(store: prism.Store*, account: str, path: str) -> !:
var identity: Buffer
defer identity.destroy()
try outbox_identity(account, &identity)
var batch = try Batch.begin(store, identity.text())
defer batch.close()
let session = try batch.session_of()
if try session.contains(path):
try session.remove(path)
batch.count += 1
try batch.commit()
try bake(store, identity.text())