Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-develop

Source of release 0.1.0

0.1.0 / dev / film / read_drawings.py

#!/usr/bin/env python3
"""Read a datasheet's vector graphs into dev/film/<stock>.json (the form
make_stocks.py reads), naming each curve by the index of its drawing on the page
(survey a page with `read_drawings.py --survey PDF PAGE`: every drawing's index,
color and box, and every text span's centre, so the tick labels give the axes).

    read_drawings.py SHEETS_DIR [STOCK ...]   # writes the JSONs and, beside them in
                                              # SHEETS_DIR/checks/, each graph's page
                                              # with what was read drawn back on it

A stock's spec (STOCKS, in any sheets_*.py beside this file) gives, per graph:
the page, two (point, value) references per axis read off its tick labels'
centres, whether an axis is logarithmic, and each curve as a drawing's index (all
its pieces), "index:piece" (one piece, as the survey numbers them),
"index@first-last" (a run of its path items, where one path draws two curves),
or a list of those joined. Needs PyMuPDF (and Pillow for the checks).

    STOCKS = {'kodak_portra_400': {
        'pdf': 'portra_400/e4050.pdf', 'name': 'Kodak Portra 400', 'source': 'Kodak E-4050 (1-25)',
        'kind': 'negative', 'ei': 400, 'kelvin': 5500.0, 'grey_log_h': -1.44,
        'characteristic': {'page': 3, 'x': ((79.6, -4.0), (266.0, 1.0)), 'y': ((287.8, 0.0), (102.9, 4.0)),
                           'curves': {'blue': 34, 'green': 35, 'red': 36}},
        ...}}

Graphs: characteristic (log H, density: red, green, blue; for black and white,
any names, such as 'd76_6.75'), 'extras' (more such graphs by name: push
curves, other developers), sensitivity (nm, log
sensitivity), dyes (nm, density: cyan, magenta, yellow, and neutral or d_min),
mtf (log c/mm, log %), granularity (log H, σD; 'shift' moves it onto the
characteristic curves' log H; or 'copy', the graph's own characteristic curves,
read on its density axis 'density_y', finds it). A stock without one gives
'rms_granularity' (diffuse rms × 1000 at density 1.0) instead."""
import json, math, sys
from pathlib import Path
import pymupdf

here = Path(__file__).resolve().parent
NM = list(range(380, 785, 5))
CHANNELS = ('red', 'green', 'blue')


def axis(ref, log):
    (p0, v0), (p1, v1) = ref
    if log:
        a, b = math.log10(v0), math.log10(v1)
        return lambda p: 10 ** (a + (p - p0) * (b - a) / (p1 - p0))
    return lambda p: v0 + (p - p0) * (v1 - v0) / (p1 - p0)


def subpaths(page, index):
    """Drawing `index`'s pieces (a new one wherever the pen lifts) as points on
    the page, curves sampled every 1/8."""
    d = page.get_drawings()[index]
    pieces, points = [], []
    for item in d['items']:
        ps = [p for p in item[1:] if hasattr(p, 'x')]
        if not ps:
            continue
        if points and (abs(points[-1][0] - ps[0].x) > 0.05 or abs(points[-1][1] - ps[0].y) > 0.05):
            pieces.append(points)
            points = []
        if not points:
            points.append((ps[0].x, ps[0].y))
        if item[0] == 'c':
            p0, p1, p2, p3 = ps
            for k in range(1, 9):
                t = k / 8
                a, b, c, e = (1 - t) ** 3, 3 * (1 - t) ** 2 * t, 3 * (1 - t) * t * t, t ** 3
                points.append((a * p0.x + b * p1.x + c * p2.x + e * p3.x, a * p0.y + b * p1.y + c * p2.y + e * p3.y))
        else:
            points.append((ps[-1].x, ps[-1].y))
    if points:
        pieces.append(points)
    return pieces


def drawing_points(page, which):
    """The points of `which`: a drawing's index (all its pieces), "index:piece",
    or "index@first-last", a run of the drawing's path items (where one
    unbroken path draws two curves)."""
    if isinstance(which, str) and '@' in which:
        index, span = which.split('@')
        first, last = (int(v) for v in span.split('-'))
        points = []
        for item in page.get_drawings()[int(index)]['items'][first:last + 1]:
            ps = [p for p in item[1:] if hasattr(p, 'x')]
            if not points:
                points.append((ps[0].x, ps[0].y))
            if item[0] == 'c':
                p0, p1, p2, p3 = ps
                for k in range(1, 9):
                    t = k / 8
                    a, b, c, e = (1 - t) ** 3, 3 * (1 - t) ** 2 * t, 3 * (1 - t) * t * t, t ** 3
                    points.append((a * p0.x + b * p1.x + c * p2.x + e * p3.x, a * p0.y + b * p1.y + c * p2.y + e * p3.y))
            else:
                points.append((ps[-1].x, ps[-1].y))
        return points
    if isinstance(which, str):
        index, piece = (int(v) for v in which.split(':'))
        return subpaths(page, index)[piece]
    return [p for piece in subpaths(page, which) for p in piece]


def curve(page, graph, which):
    """Curve `which` of `graph` in axis units, x rising."""
    indices = which if isinstance(which, list) else [which]
    X = axis(graph['x'], graph.get('log_x', False))
    Y = axis(graph['y'], graph.get('log_y', False))
    pts = []
    for i in indices:
        pts += [(X(px), Y(py)) for px, py in drawing_points(page, i)]
    return sorted(pts)


def sampled(pts, xs):
    out = []
    for q in xs:
        v = None
        for (a, ya), (b, yb) in zip(pts, pts[1:]):
            if a <= q <= b:
                v = ya if b == a else ya + (yb - ya) * (q - a) / (b - a)
                break
        out.append(None if v is None else round(v, 4))
    return out


def steps(first, last, step):
    n = int(math.floor((last - first) / step + 1e-9)) + 1
    return [round(first + step * i, 4) for i in range(n)]


def read(folder, spec, checks):
    doc = pymupdf.open(folder / spec['pdf'])
    out = {k: spec[k] for k in ('name', 'source', 'ei', 'kelvin', 'grey_log_h', 'kind') if k in spec}
    out['source'] = spec['source'] + ', vector graphs read by dev/film/read_drawings.py'
    marks = {}
    def take(graph):
        page = doc[graph['page']]
        curves = {name: curve(page, graph, which) for name, which in graph['curves'].items()}
        marks.setdefault(graph['page'], []).append((graph, curves))
        return curves
    def over_log_h(graph):
        """A graph of density over log H, every 0.05 where any curve is read."""
        g = take(graph)
        first = min(min(p[0] for p in pts) for pts in g.values())
        last = max(max(p[0] for p in pts) for pts in g.values())
        grid = steps(math.ceil(first / 0.05) * 0.05, last, 0.05)
        return {'log_h': grid, **{c: sampled(g[c], grid) for c in g}}
    out['characteristic'] = over_log_h(spec['characteristic'])
    # Further density curves (other developers or times, push curves), by name.
    for name, graph in spec.get('extras', {}).items():
        out[name] = over_log_h(graph)
    if 'sensitivity' in spec:
        g = take(spec['sensitivity'])
        out['sensitivity'] = {'nm': NM, **{c: sampled(g[c], NM) for c in g}}
    if 'dyes' in spec:
        g = take(spec['dyes'])
        out['dyes'] = {'nm': NM, **{d: sampled(g[d], NM) for d in g if d in ('cyan', 'magenta', 'yellow')}}
        for extra in ('neutral', 'd_min'):
            if extra in g:
                out[extra] = {'nm': NM, 'density': sampled(g[extra], NM)}
    if 'mtf' in spec:
        g = take(spec['mtf'])
        top = min(max(p[0] for p in g[c]) for c in g)
        freqs = [f for f in (2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 25, 30, 40, 50, 60, 70, 80, 100, 125, 150, 200) if f <= top
                 and all(min(p[0] for p in g[c]) <= f for c in g)]
        out['mtf'] = {'cycles_per_mm': freqs, **{c: sampled(g[c], freqs) for c in g}}
    if 'granularity' in spec:
        g = take(spec['granularity'])
        lows = max(min(p[0] for p in g[c]) for c in g)
        highs = min(max(p[0] for p in g[c]) for c in g)
        grid = steps(math.ceil(lows / 0.1) * 0.1, highs, 0.1)
        shift = spec['granularity'].get('shift', 0.0)
        if 'copy' in spec['granularity']:
            shift = aligned(doc, spec['granularity'], out['characteristic'])
        out['granularity'] = {'log_h': [round(q + shift, 4) for q in grid], **{c: sampled(g[c], grid) for c in g}}
    for k in ('rms_granularity', 'note'):
        if k in spec:
            out[k] = spec[k]
    if checks is not None:
        draw_checks(doc, marks, checks / spec['key'])
    return out


def aligned(doc, graph, main):
    """The log H shift that lays the granularity graph's own copy of the
    characteristic curves ('copy', read on its density axis 'density_y') over the
    main ones, to 0.01."""
    page = doc[graph['page']]
    copy_graph = {**graph, 'y': graph['density_y'], 'log_y': False}
    copies = {c: curve(page, copy_graph, which) for c, which in graph['copy'].items()}
    best, shift = 1e9, 0.0
    for k in range(-600, 601):
        s = k * 0.01
        err, n = 0.0, 0
        for c, pts in copies.items():
            moved = [(x + s, y) for x, y in pts]
            for q, v in zip(main['log_h'], sampled(moved, main['log_h'])):
                if v is not None and main[c][main['log_h'].index(q)] is not None:
                    err += (v - main[c][main['log_h'].index(q)]) ** 2
                    n += 1
        if n > 30 and err / n < best:
            best, shift = err / n, s
    print(f"  granularity shift {shift:+.2f} (rms {best ** 0.5:.3f} density)")
    return shift


def draw_checks(doc, marks, stem):
    """Each page read, at 3×, with what was read drawn back on it in magenta."""
    from PIL import Image, ImageDraw
    stem.parent.mkdir(parents=True, exist_ok=True)
    for number, graphs in marks.items():
        pix = doc[number].get_pixmap(matrix=pymupdf.Matrix(3, 3))
        image = Image.frombytes('RGB', (pix.width, pix.height), pix.samples)
        draw = ImageDraw.Draw(image)
        for graph, curves in graphs:
            def back(ref, log, value):
                (p0, v0), (p1, v1) = ref
                if log:
                    v0, v1, value = math.log10(v0), math.log10(v1), math.log10(max(value, 1e-9))
                return p0 + (value - v0) * (p1 - p0) / (v1 - v0)
            for pts in curves.values():
                for x, y in pts[::2]:
                    px = back(graph['x'], graph.get('log_x', False), x) * 3
                    py = back(graph['y'], graph.get('log_y', False), y) * 3
                    draw.ellipse((px - 2, py - 2, px + 2, py + 2), fill=(255, 0, 200))
        image.save(f'{stem}-p{number}.png')


def survey(pdf, number):
    page = pymupdf.open(pdf)[number]
    for i, d in enumerate(page.get_drawings()):
        r = d['rect']
        if len(d['items']) < 3:
            continue
        color = tuple(round(c, 2) for c in d['color']) if d.get('color') else None
        print(f"D{i} color={color} box=({r.x0:.1f},{r.y0:.1f},{r.x1:.1f},{r.y1:.1f}) items={len(d['items'])}")
        pieces = subpaths(page, i)
        if len(pieces) > 1:
            for k, piece in enumerate(pieces):
                xs = [p[0] for p in piece]; ys = [p[1] for p in piece]
                if len(piece) >= 4:
                    print(f"   {i}:{k} x {min(xs):.1f}..{max(xs):.1f} y {min(ys):.1f}..{max(ys):.1f} n={len(piece)}")
    for block in page.get_text('dict')['blocks']:
        for line in block.get('lines', []):
            for span in line['spans']:
                if span['text'].strip():
                    x0, y0, x1, y1 = span['bbox']
                    print(f"T ({(x0 + x1) / 2:.1f},{(y0 + y1) / 2:.1f}) {span['text'].strip()!r}")


if __name__ == '__main__':
    if sys.argv[1] == '--survey':
        survey(sys.argv[2], int(sys.argv[3]))
        sys.exit()
    import importlib
    sys.path.insert(0, str(here))
    stocks = {}
    for path in sorted(here.glob('sheets_*.py')):
        stocks.update(importlib.import_module(path.stem).STOCKS)
    folder = Path(sys.argv[1])
    wanted = sys.argv[2:] or list(stocks)
    for key in wanted:
        spec = {**stocks[key], 'key': key}
        data = read(folder, spec, folder / 'checks')
        (here / f'{key}.json').write_text(json.dumps(data, indent=1))
        c = data['characteristic']
        print(f"{key}: log H {c['log_h'][0]}..{c['log_h'][-1]}" + (f", MTF to {data['mtf']['cycles_per_mm'][-1]}" if 'mtf' in data else ''))