dymokomi/luced-photo
Source of release 0.1.0
0.1.0 / src / import_worker.lucb
## The import's workers: several threads take the gathered files in turn and do
## what is slow — reading, hashing, the metadata, copying the original, decoding
## the preview, encoding the preview and thumbnail files — at once. What must
## be decided one file at a time is decided under the job's lock: whether the
## photo is already in the library (by SHA-256, its own import included), the
## photo's id, and its file name in originals/ (no two take one name). Each file
## ends in a Result the import thread commits to the catalog (importer.lucb).
## Files may still be arriving while the workers run (downloaded from a camera):
## a worker with none left waits for the next, and removes a staged file (and the
## folder it was staged in) once it is through with it.
import c
import memory
import strings
import sync
import thread
import time
from luce_std import files
from luce_std import paths
from luce_std.collections import Buffer, List, TextPool
from luce_crypto import native as crypto
from luce_develop import source
import catalog
from catalog import Photo
import facts
import library
## How a file went.
pub enum Outcome:
added
skipped
failed
## One file's end: the photo added (its text in the job's pool), or why not.
pub struct Result:
pub var file: usize
pub var outcome: Outcome
pub var photo: Photo
pub var message: str
## What the workers share; everything marked `under lock` only under `lock`.
pub struct Job:
pub var lock: sync.Mutex
pub var ready: sync.Condition
pub var root: str
## Under lock: the files to import (text in `paths`), the next one a worker
## takes, and whether more may still arrive; `more` is signalled when they do.
pub var files: List[str]
pub var paths: TextPool
pub var next_file: usize
pub var arriving: bool
pub var more: sync.Condition
## The files are staged copies, removed once imported.
pub var staged: bool
## Under lock: the library's photos before the import (for their hashes),
## the hashes and file names this import has claimed, the next id, results
## not taken yet, how many files are finished, and the text of both.
pub var known: catalog.Rows
pub var hashes: List[str]
pub var names: List[str]
pub var next_id: i64
pub var results: List[Result]
pub var finished: usize
pub var pool: TextPool
pub var cancelled: @bool
func holds(hash: str) -> bool:
for photo in self.known.photos.view():
if photo.sha256 == hash:
return true
for kept in self.hashes.view():
if kept == hash:
return true
return false
func claimed(relative: str) -> bool:
for kept in self.names.view():
if kept == relative:
return true
return false
## Add `path` to the files to import and wake a waiting worker.
pub func arrive(path: str) -> !:
self.lock.lock()
defer self.lock.unlock()
try self.files.append(try self.paths.keep_text(path.bytes))
self.more.signal()
## No more files will arrive (or the import is cancelled): wake every worker.
pub func arrived():
self.lock.lock()
self.arriving = false
self.more.broadcast()
self.lock.unlock()
## File `at`'s path.
pub func file(at: usize) -> str:
self.lock.lock()
defer self.lock.unlock()
return self.files.at(at)
## The next file to import, waiting while more may arrive; none when there
## are no more or the import is cancelled.
func next() -> usize?:
self.lock.lock()
defer self.lock.unlock()
while self.next_file >= self.files.length() and self.arriving and not self.cancelled.load(.acquire):
self.more.wait(&self.lock)
if self.next_file >= self.files.length() or self.cancelled.load(.acquire):
return none
let at = self.next_file
self.next_file += 1
return at
pub func close():
self.files.destroy()
self.paths.destroy()
self.known.close()
self.hashes.destroy()
self.names.destroy()
self.results.destroy()
self.pool.destroy()
## A worker: take files until there are none, or the import is cancelled.
pub func work(context: void*?):
let job = (Job*)(context else return)
while true:
let at = job.next() else return
let file = job.file(at)
var local: TextPool
defer local.destroy()
var result = Result(file = at, outcome = .failed)
# The failure's message lives only as long as its handler; finish() keeps a copy of this one.
var held: u8[256]
result.outcome = import_file(job, file, &result.photo, &local) catch failure:
result.message = strings.copy_into(held, failure.message)
recover Outcome.failed
if job.staged:
unstage(file)
job.lock.lock()
finish(job, &result)
job.lock.unlock()
job.ready.signal()
## Remove the staged copy at `path` and the folder it was staged in; what cannot
## be removed stays for the next camera import to clear.
func unstage(path: str):
let file = strings.copy(path) catch failure:
return
defer strings.release(file)
files.remove_file((c.str)file) catch failure:
return
let folder = strings.copy(paths.directory(path)) catch failure:
return
defer strings.release(folder)
files.remove_directory((c.str)folder) catch failure:
_ = failure
## Keep `result`'s text in the job's pool and hand it to the import thread (under lock).
func finish(job: Job*, result: Result*):
result.message = job.pool.keep_text(result.message.bytes) catch failure:
recover ""
if result.outcome == Outcome.added:
keep_photo(&result.photo, &job.pool) catch failure:
result.outcome = .failed
result.message = "out of memory"
job.results.append(*result) catch failure:
_ = failure
job.finished += 1
func keep_photo(photo: Photo*, pool: TextPool*) -> !:
photo.path = try pool.keep_text(photo.path.bytes)
photo.name = try pool.keep_text(photo.name.bytes)
photo.kind = try pool.keep_text(photo.kind.bytes)
photo.sha256 = try pool.keep_text(photo.sha256.bytes)
photo.captured = try pool.keep_text(photo.captured.bytes)
photo.offset = try pool.keep_text(photo.offset.bytes)
photo.make = try pool.keep_text(photo.make.bytes)
photo.model = try pool.keep_text(photo.model.bytes)
photo.lens = try pool.keep_text(photo.lens.bytes)
photo.shutter = try pool.keep_text(photo.shutter.bytes)
photo.settings = try pool.keep_text(photo.settings.bytes)
## Import the file at `file` into `photo`, its text in `local`.
func import_file(job: Job*, file: str, photo: Photo*, local: TextPool*) -> Outcome!:
let terminated = try strings.copy(file)
defer strings.release(terminated)
let data = try files.read((c.str)terminated)
defer data.release()
let found = source.kind(data.value)
if found == source.Kind.unknown:
error(source.unsupported, "not a photo luced-photo reads")
var digest: u8[32]
_ = try crypto.digest(data.value, digest, "sha256")
var hex: u8[64]
for (at, byte) in digest.indexed():
hex[at * 2] = hex_digit(byte >> 4)
hex[at * 2 + 1] = hex_digit(byte & 15)
*photo = Photo(imported = time.unix())
photo.sha256 = try local.keep_text(hex)
try facts.describe(file, data.value, found, photo, local)
# The original's folder, by capture date: originals/YYYY/MM/DD/.
var relative: Buffer
defer relative.destroy()
try relative.put(b"originals/")
try relative.put(photo.captured.bytes[0..<4])
try relative.put_byte('/')
try relative.put(photo.captured.bytes[5..<7])
try relative.put_byte('/')
try relative.put(photo.captured.bytes[8..<10])
var path: Buffer
defer path.destroy()
try library.join_into(job.root, relative.text(), &path)
try files.ensure_directory(path.text())
try relative.put_byte('/')
# One file at a time: the duplicate check, the id and the name.
job.lock.lock()
let claim = claim_name(job, photo, &relative, &path) catch failure:
job.lock.unlock()
error(failure.code, failure.message)
job.lock.unlock()
if not claim:
return .skipped
photo.path = try local.keep_text(relative.view())
try library.join_into(job.root, photo.path, &path)
try files.write_atomic(try path.terminated(), data.value)
# The viewer's preview and the grid's thumbnail.
var preview = try source.preview(data.value, (usize)library.preview_edge, (u64)photo.orientation)
defer preview.close()
if photo.width == 0:
photo.width = preview.width()
photo.height = preview.height()
try library.cache_path(job.root, "previews", photo.id, &path)
try preview.save_jpeg(path.text(), 88)
var thumb = try preview.shrunk((usize)library.thumb_edge)
defer thumb.close()
try library.cache_path(job.root, "thumbs", photo.id, &path)
try thumb.save_jpeg(path.text(), 82)
return .added
## Under lock: false when the library already holds the photo; else claim its
## hash, an id and a free name, appended to `relative` (a folder ending in /):
## the name itself, else name-2.ext, name-3.ext…
func claim_name(job: Job*, photo: Photo*, relative: Buffer*, path: Buffer*) -> bool!:
if job.holds(photo.sha256):
return false
try job.hashes.append(try job.pool.keep_text(photo.sha256.bytes))
photo.id = job.next_id
job.next_id += 1
let folder = relative.length()
let stem = paths.stem(photo.name)
let extension = paths.extension(photo.name)
var attempt: i64 = 1
while true:
relative.truncate(folder)
try relative.put_text(stem)
if attempt > 1:
try relative.put_byte('-')
try relative.put_number(attempt)
try relative.put_text(extension)
try library.join_into(job.root, relative.text(), path)
if not job.claimed(relative.text()) and not files.exists(try path.terminated()):
try job.names.append(try job.pool.keep_text(relative.view()))
return true
attempt += 1
func hex_digit(value: u8) -> u8:
return value + '0' if value < 10 else value - 10 + 'a'
## How many workers an import of `count` files runs: a few, since each holds a
## whole file and the JPEG coders spread over the processors themselves.
pub func worker_count(count: usize) -> usize:
let most: usize = 4
return count if count < most else most