dymokomi/luced-photo
Source of release 0.1.0
## The importer: copies photos into the library on threads of its own, so the
## window never waits on a card, a disk or a camera. Its thread gathers the files
## under the paths it was given, or lists a camera's photos over PTP (luce-ptp)
## and downloads the ones the library lacks into incoming/ one by one, and starts
## a few workers (import_worker.lucb) that take the files as they come; each file is
## read, hashed (a photo the library holds is skipped), described, copied to
## originals/YYYY/MM/DD/ by capture date and given its preview and thumbnail.
## This thread commits the photos' rows as the workers finish them, a few at a
## time, so the grid fills in as the import goes.
##
## The window polls events, one line each, fields separated by tabs:
## found total files to look at
## fetching name done total copying a photo from the camera
## added id done total name a photo is in the library
## skipped name done total already in the library
## failed name message done total could not be imported
## done added skipped failed why the import is over (why it stopped
## early, or empty)
import c
import interop
import memory
import strings
import sync
import thread
import time
from luce_std import files
from luce_std import paths
from luce_prism import prism
from luce_crypto import native as crypto
from luce_develop import source
import catalog
import import_worker
from import_worker import Job, Result
from luce_std.collections import Buffer, TextPool, List
import facts
import library
from luce_ui import ui
from luce_ptp import ptp
## How many photos each commit carries.
let commit_every: usize = 6
## What the window and the import thread share, guarded by `lock`.
struct Room:
var lock: sync.Mutex
var events: Buffer
var cancelled: bool
var finished: bool
## The bundle root and the paths to import, one a line; set before the start.
var root: Buffer
var sources: Buffer
var store: prism.Store*?
## The library folder the photos go into (0 none).
var folder: i64
## The camera to import from instead of `sources` (the import thread takes it).
var camera: ptp.Transport?
## The window's notifier, told of every event and of the end (none: polled
## by a caller with no loop, as the tests are).
var waker: ui.NotifyHandle?
func post(line: const u8[]):
self.lock.lock()
defer self.lock.unlock()
self.events.put(line) catch failure:
return
self.events.put_byte('\n') catch failure:
return
if let waker = self.waker:
waker.notify()
func drain(out: Buffer*) -> !:
self.lock.lock()
defer self.lock.unlock()
try out.put(self.events.view())
self.events.clear()
func cancel():
self.lock.lock()
self.cancelled = true
self.lock.unlock()
func is_cancelled() -> bool:
self.lock.lock()
defer self.lock.unlock()
return self.cancelled
func finish():
self.lock.lock()
self.finished = true
if let waker = self.waker:
waker.notify()
self.lock.unlock()
func is_finished() -> bool:
self.lock.lock()
defer self.lock.unlock()
return self.finished
func close():
if let left = self.camera:
left.close(left.context)
self.camera = none
self.events.destroy()
self.root.destroy()
self.sources.destroy()
if let waker = self.waker:
waker.release()
self.waker = none
## The import thread: import every file the room names, then report.
func serve(context: void*?):
let room = (Room*)(context else return)
work(room) catch failure:
var line: Buffer
defer line.destroy()
line.put(b"error\t") catch dropped:
_ = dropped
put_field(&line, failure.message.bytes) catch dropped:
_ = dropped
room.post(line.view())
room.finish()
## Counts and lines of the import as the workers' results arrive.
struct Tally:
var added: i64
var skipped: i64
var failed: i64
var taken: usize
## The id after the newest photo committed.
var next: i64
var line: Buffer
## "added" lines wait here until their rows are committed, so the window can read them.
var pending: Buffer
## Why the import stopped before its end (a camera unplugged), or empty.
var stopped: Buffer
func close():
self.line.destroy()
self.pending.destroy()
self.stopped.destroy()
func work(room: Room*) -> !:
let store = room.store else error(prism.closed, "the importer has no catalog")
let root = room.root.text()
try library.ensure(root)
var job = Job(root = root, known = try catalog.read(store), arriving = true)
defer job.close()
job.next_id = job.known.next
var tally: Tally
defer tally.close()
tally.next = job.next_id
var batch = try catalog.Batch.begin(store)
defer batch.close()
if let transport = room.camera:
room.camera = none
job.staged = true
var camera = try ptp.Camera.open(transport)
defer camera.close()
try import_all(room, &job, &tally, &batch, store, &camera)
else:
try gather(room.sources.text(), root, &job)
job.arriving = false
try import_all(room, &job, &tally, &batch, store, none)
try batch.commit()
try post_lines(room, &tally.pending)
try catalog.bake(store)
tally.line.clear()
try tally.line.put(b"done\t")
try tally.line.put_number(tally.added)
try tally.line.put_byte('\t')
try tally.line.put_number(tally.skipped)
try tally.line.put_byte('\t')
try tally.line.put_number(tally.failed)
try tally.line.put_byte('\t')
try put_field(&tally.line, tally.stopped.view())
room.post(tally.line.view())
## Import the job's files, or `camera`'s photos the library lacks as they download:
## post how many there are, start the workers and commit what they finish.
func import_all(room: Room*, job: Job*, tally: Tally*, batch: catalog.Batch*, store: prism.Store*, camera: ptp.Camera*?) -> !:
var wanted: List[Wanted]
defer wanted.destroy()
var listed: TextPool
defer listed.destroy()
var total = job.files.length()
if let open = camera:
try list_camera(open, job, &wanted, &listed)
total = wanted.length()
tally.line.clear()
try tally.line.put(b"found\t")
try tally.line.put_number((i64)total)
room.post(tally.line.view())
var workers: thread.Handle[4]
var started: usize = 0
for _at in 0..<import_worker.worker_count(total):
workers[started] = thread.spawn(import_worker.work, (void*)job, name = "luced-photo import worker") catch failure:
break
started += 1
if let open = camera:
total = fetch_camera(open, job, tally, batch, store, room, wanted.view(), started > 0) catch failure:
job.cancelled.store(true, .release)
stop_workers(job, workers[..<started])
error(failure.code, failure.message)
job.arrived()
while tally.taken < total and started > 0:
if room.is_cancelled():
job.cancelled.store(true, .release)
job.arrived()
break
try take_results(job, tally, batch, store, room, total, true)
# Workers end with the file in hand; what they finished is still committed.
stop_workers(job, workers[..<started])
try take_results(job, tally, batch, store, room, total, false)
## Wake the workers waiting for files that will not come, and wait for them.
func stop_workers(job: Job*, workers: thread.Handle[]):
job.arrived()
for worker in workers:
worker.join() catch failure:
_ = failure
# mark: from a camera ===========================================================================
## A photo on the camera: its handle, size and name (text in the listing's pool),
## and whether the library holds it already (by name and size).
struct Wanted:
var handle: u32
var size: u32
var name: str
var held: bool
## Every photo on `camera`, onto `wanted`.
func list_camera(camera: ptp.Camera*, job: Job*, wanted: List[Wanted]*, pool: TextPool*) -> !:
var handles: List[u32]
defer handles.destroy()
try camera.handles(&handles)
for handle in handles.view():
let info = try camera.object_info(handle, pool)
if info.is_folder() or not source.is_photo_name(info.name):
continue
try wanted.append(Wanted(handle = handle, size = info.size, name = info.name, held = holds_file(job, info.name, info.size)))
## Whether the library holds a photo imported from a file of this name and size.
func holds_file(job: Job*, name: str, size: u32) -> bool:
for photo in job.known.photos.view():
if photo.bytes == (i64)size and photo.name == name:
return true
return false
## Download each wanted photo the library lacks into incoming/<handle>/ and hand it
## to the workers, committing what they finish between downloads. When the camera
## goes or a download stops, the photos not reached are left out: the total the
## import now waits for, returned, counts only the ones reached.
func fetch_camera(camera: ptp.Camera*, job: Job*, tally: Tally*, batch: catalog.Batch*, store: prism.Store*, room: Room*,
wanted: const Wanted[], working: bool) -> usize!:
var incoming: Buffer
defer incoming.destroy()
try library.join_into(job.root, "incoming", &incoming)
# What an earlier import left behind (it stopped before its workers were through).
files.remove_tree(try incoming.terminated()) catch failure:
_ = failure
try files.ensure_directory(incoming.text())
var fetching: i64 = 0
for item in wanted:
if not item.held:
fetching += 1
var fetched: i64 = 0
var staged: Buffer
defer staged.destroy()
for (at, item) in wanted.indexed():
if room.is_cancelled():
return at
if item.held:
tally.taken += 1
tally.skipped += 1
tally.line.clear()
try tally.line.put(b"skipped\t")
try put_field(&tally.line, item.name.bytes)
try progress_into(&tally.line, tally.taken, wanted.length)
room.post(tally.line.view())
continue
fetched += 1
tally.line.clear()
try tally.line.put(b"fetching\t")
try put_field(&tally.line, item.name.bytes)
try progress_into(&tally.line, (usize)fetched, (usize)fetching)
room.post(tally.line.view())
staged.clear()
try staged.put(incoming.view())
try staged.put_byte('/')
try staged.put_number((i64)item.handle)
try files.ensure_directory(staged.text())
try staged.put_byte('/')
try staged.put_text(item.name)
camera.download(item.handle, item.size, staged.text(), ptp.Progress(call = keep_going, context = (void*)room)) catch failure:
if failure.code == ptp.cancelled or failure.code == ptp.disconnected:
tally.stopped.clear()
try tally.stopped.put_text(failure.message)
return at
tally.taken += 1
tally.failed += 1
tally.line.clear()
try tally.line.put(b"failed\t")
try put_field(&tally.line, item.name.bytes)
try tally.line.put_byte('\t')
try put_field(&tally.line, failure.message.bytes)
try progress_into(&tally.line, tally.taken, wanted.length)
room.post(tally.line.view())
continue
try job.arrive(staged.text())
if working:
try take_results(job, tally, batch, store, room, wanted.length, false)
return wanted.length
## A download goes on while the import is not cancelled.
func keep_going(context: void*?, done: u64, total: u64) -> bool:
let room = (Room*)(context else return false)
return not room.is_cancelled()
## Take the workers' finished files (waiting for one when `wait`), commit the
## added photos' rows a few at a time and post what happened.
func take_results(job: Job*, tally: Tally*, batch: catalog.Batch*, store: prism.Store*, room: Room*, total: usize, wait: bool) -> !:
var results: List[Result]
defer results.destroy()
job.lock.lock()
while wait and job.results.length() == 0:
job.ready.wait(&job.lock)
for result in job.results.view():
results.append(result) catch failure:
job.lock.unlock()
error(failure.code, failure.message)
job.results.clear()
job.lock.unlock()
for result in results.view():
tally.taken += 1
let name = paths.base(job.file(result.file))
tally.line.clear()
match result.outcome:
.failed:
tally.failed += 1
try tally.line.put(b"failed\t")
try put_field(&tally.line, name.bytes)
try tally.line.put_byte('\t')
try put_field(&tally.line, result.message.bytes)
try progress_into(&tally.line, tally.taken, total)
room.post(tally.line.view())
.skipped:
tally.skipped += 1
try tally.line.put(b"skipped\t")
try put_field(&tally.line, name.bytes)
try progress_into(&tally.line, tally.taken, total)
room.post(tally.line.view())
.added:
tally.added += 1
var photo = result.photo
photo.folder = room.folder
try batch.put(&photo)
if result.photo.id >= tally.next:
tally.next = result.photo.id + 1
try batch.set_next(tally.next)
try tally.pending.put(b"added\t")
try tally.pending.put_number(result.photo.id)
try progress_into(&tally.pending, tally.taken, total)
try tally.pending.put_byte('\t')
try put_field(&tally.pending, name.bytes)
try tally.pending.put_byte('\n')
if batch.count >= commit_every:
try batch.flush(store)
try post_lines(room, &tally.pending)
func progress_into(line: Buffer*, done: usize, total: usize) -> !:
try line.put_byte('\t')
try line.put_number((i64)done)
try line.put_byte('\t')
try line.put_number((i64)total)
## Post every line of `pending` and empty it.
func post_lines(room: Room*, pending: Buffer*) -> !:
if pending.length() == 0:
return
# Without its last newline: post adds one.
room.post(pending.view()[..<pending.length() - 1])
pending.clear()
## Every photo file named in `sources` (one path a line), onto the job's files:
## files as they are, folders walked, files inside the library at `root` left out.
func gather(sources: str, root: str, job: Job*) -> !:
var first: usize = 0
let text = sources.bytes
while first < text.length:
var end = first
while end < text.length and text[end] != '\n':
end += 1
let path = (str)text[first..<end]
first = end + 1
if path.length == 0 or path.bytes.length >= root.length and (str)path.bytes[..<root.length] == root:
continue
let terminated = try strings.copy(path)
defer strings.release(terminated)
let stat = files.metadata((c.str)terminated) catch failure:
continue
if stat.kind == files.FileKind.directory:
let entries = try files.walk((c.str)terminated)
defer entries.release()
for entry in entries.value:
if source.is_photo_name(entry):
let joined = try paths.join(path, entry)
defer joined.release()
try job.arrive(joined.value)
elif source.is_photo_name(path):
try job.arrive(path)
## Append `data` as one tab-separated field: tabs and line breaks inside it become
## spaces, so the window may split event lines at tabs.
func put_field(out: Buffer*, data: const u8[]) -> !:
for byte in data:
try out.put_byte(' ' if byte == '\t' or byte == '\n' or byte == '\r' else byte)
## A running import.
pub struct Importer:
var room: Room*?
var handle: thread.Handle
## Keeps the catalog the thread writes open until the thread has stopped.
var store: interop.Reference[prism.Store]?
var polled: Buffer
## The events since the last poll, one a line; empty when there are none.
pub func poll() -> str!:
self.polled.clear()
let room = self.room else return ""
try room.drain(&self.polled)
return self.polled.text()
## Whether the import is over (its last events may still wait for a poll).
pub func finished() -> bool:
let room = self.room else return true
return room.is_finished()
## Stop after the file being imported; what is in the library stays.
pub func cancel():
if let room = self.room:
room.cancel()
## Stop the import and wait for its thread.
pub func close():
let room = self.room else return
room.cancel()
self.handle.join() catch failure:
trap("the import thread could not be joined")
room.close()
free(room) in memory.heap
self.room = none
self.polled.destroy()
if let held = self.store:
held.release()
self.store = none
pub let importer_type: interop.Type[Importer] = interop.Type[Importer]("Importer", Importer.close, closeable = true, main_thread = true)
## Import the photo files and folders `sources` into the library bundle at
## `root`, whose catalog is `store`, on a thread of its own; the photos go into
## library folder `folder` (0 none); `notifier` hears of every event (none:
## the caller polls).
pub func start(store: interop.Reference[prism.Store], root: str, sources: const str[], folder: i64,
notifier: interop.Reference[ui.Notifier]? = none) -> interop.Reference[Importer]!:
let room = try new Room in memory.heap
errdefer free(room) in memory.heap
for path in sources:
try room.sources.put_text(path)
try room.sources.put_byte('\n')
return try begin(room, store, root, folder, notifier)
## Import the photos on camera `at` of `watcher` that the library lacks, as
## `start` imports files; the camera is connected here, on the main thread, and
## read on the import thread.
pub func start_camera(store: interop.Reference[prism.Store], root: str, watcher: interop.Reference[ptp.Watcher], at: i64, folder: i64,
notifier: interop.Reference[ui.Notifier]? = none) -> interop.Reference[Importer]!:
return try start_transport(store, root, try (try watcher.get()).connect(at), folder, notifier)
## Import the photos the library lacks from the camera over `transport` (which the
## importer then owns): a camera a watcher connected, or a virtual one.
pub func start_transport(store: interop.Reference[prism.Store], root: str, transport: ptp.Transport, folder: i64,
notifier: interop.Reference[ui.Notifier]? = none) -> interop.Reference[Importer]!:
let room = new Room(camera = transport) in memory.heap catch failure:
transport.close(transport.context)
error(failure.code, failure.message)
errdefer discard(room)
return try begin(room, store, root, folder, notifier)
## Let a room no thread took go.
func discard(room: Room*):
room.close()
free(room) in memory.heap
## Start the import thread over `room`, which it then owns.
func begin(room: Room*, store: interop.Reference[prism.Store], root: str, folder: i64,
notifier: interop.Reference[ui.Notifier]?) -> interop.Reference[Importer]!:
room.folder = folder
if let told = notifier:
room.waker = try ui.notify_handle(told)
try room.root.put_text(root)
room.store = try store.get()
let handle = try thread.spawn(serve, (void*)room, name = "luced-photo import")
let importer = try new Importer(room = room, handle = handle, store = store.clone()) in memory.heap
return try interop.Reference[Importer].adopt(importer, importer_type, memory.heap)