dymokomi/luce-raw
Source of release 0.8.2
0.8.2 / src / raw / xtrans_steps.lucb
#==============================================================================================
#
# raw/xtrans_steps - Red, blue and the choice of direction for X-Trans
#
# DESCRIPTION:
# The later steps of LibRaw's xtrans_interpolate on one tile: red and blue
# for solitary greens (choosing between diagonal estimates by their
# smoothness), red for blue pixels and blue for red, red and blue for 2×2
# green blocks, and the final choice: CIELAB derivatives along each
# direction, homogeneity counts, and the mean of the best directions.
# `rgb` holds eight (or, during refinement, four) directions of RGB.
#
#==============================================================================================
inline func tile_index(row: i64, col: i64, t: i64, l: i64) -> i64:
return ((row - t) * (i64)xtrans_side + col - l) * 3
## `value // 2` (truncating) without a division.
inline func half32(value: i32) -> i32:
return (value +% (i32)((u32)value >> 31)) >> 1
## `value // 3` (truncating) as a multiply: exact for every i32.
inline func third32(value: i32) -> i32:
return (i32)(((i64)value *% 1431655766) >> 32) +% (i32)((u32)value >> 31)
## `value // divisor` (truncating) for |value| < 2²⁵ as a multiply by
## `reciprocal`, ⌈2⁴⁰ / divisor⌉: exact for divisors below 512.
inline func divide40(value: i32, reciprocal: i64) -> i32:
let sign = value >> 31
let size = (i64)((value ^ sign) -% sign)
let quotient = (i32)((size *% reciprocal) >> 40)
return (quotient ^ sign) -% sign
## |value| without a branch.
inline func magnitude32(value: i32) -> i32:
let sign = value >> 31
return (value ^ sign) -% sign
## Red and blue for the solitary green pixels, in four directions.
func xtrans_solitary(plane: const Plane*, rings: const Hexagons*, rgb: u16[], t: i64, l: i64, mrow: i64, mcol: i64):
let ts = (i64)xtrans_side
let step = (usize)(ts * ts * 3)
let sg_row = (i64)rings.solitary_row
let sg_col = (i64)rings.solitary_col
let data = rgb.data
var row = (t - sg_row + 4) // 3 * 3 + sg_row
while row < mrow - 2:
var col = (l - sg_col + 4) // 3 * 3 + sg_col
while col < mcol - 2:
var rix = data + (usize)tile_index(row, col, t, l)
var h = (usize)xcolor(plane, col + 1, row)
if h == 1:
break
var diff: f32[6]
var color: i32[24]
var i: usize = 1
for d: usize in 0..<6:
for c: usize in 0..<2:
let off = (i << c) * 3
let ahead = rix + off
let behind = rix - off
let g = 2 *% (i32)rix[1] -% (i32)ahead[1] -% (i32)behind[1]
color[h * 8 + d] = g +% (i32)ahead[h] +% (i32)behind[h]
if d > 1:
let e = (f32)ahead[1] - (f32)behind[1] - (f32)ahead[h] + (f32)behind[h]
diff[d] += e * e + (f32)g * (f32)g
h ^= 2
if d > 1 and (d & 1) == 1 and diff[d - 1] < diff[d]:
color[d] = color[d - 1]
color[16 + d] = color[16 + d - 1]
if d < 2 or (d & 1) == 1:
rix[0] = clamp16((i64)(color[d] // 2))
rix[2] = clamp16((i64)(color[16 + d] // 2))
rix = rix + step
i ^= (usize)ts ^ 1
h ^= 2
col += 3
row += 3
## Red for blue pixels and blue for red ones.
func xtrans_opposite(plane: const Plane*, rings: const Hexagons*, rgb: u16[], t: i64, l: i64, mrow: i64, mcol: i64):
let ts = xtrans_side
let step = ts * ts * 3
let data = rgb.data
for row in t + 3..<mrow - 3:
let colors = &plane.pattern[(usize)(row % 6) * 6]
let c = ts if (row - (i64)rings.solitary_row) % 3 != 0 else 1
let h = 3 * (c ^ ts ^ 1)
# Offsets of the two candidate neighbours' green; directions 0 and 1
# may take `h`, 2 and 3 always take `c`.
let c_ahead = (isize)(c * 3 + 1)
let c_behind = 1 - (isize)(c * 3)
let h_ahead = (isize)(h * 3 + 1)
let h_behind = 1 - (isize)(h * 3)
let fixed0 = (c & 1) != 0
let fixed1 = ((1 ^ c) & 1) != 0
var phase = (usize)((l + 3) % 6)
for col in l + 3..<mcol - 3:
let f = 2 - (usize)colors[phase]
phase = 0 if phase == 5 else phase + 1
if f == 1:
continue
let q = data + (usize)tile_index(row, col, t, l)
opposite_step(q, f, c_ahead, c_behind, h_ahead, h_behind, fixed0)
opposite_step(q + step, f, c_ahead, c_behind, h_ahead, h_behind, fixed1)
opposite_step(q + 2 * step, f, c_ahead, c_behind, h_ahead, h_behind, true)
opposite_step(q + 3 * step, f, c_ahead, c_behind, h_ahead, h_behind, true)
## One direction of `xtrans_opposite` at `q`: color `f` from the neighbours
## along `c` (or along `h` where that is smoother and not `fixed`), each
## offset given to the green sample.
inline func opposite_step(q: u16*, f: usize, c_ahead: isize, c_behind: isize, h_ahead: isize, h_behind: isize, fixed: bool):
let g0 = (i32)q[1]
var ahead = c_ahead
var behind = c_behind
if not fixed:
let across = magnitude32(g0 -% (i32)*(q + c_ahead)) +% magnitude32(g0 -% (i32)*(q + c_behind))
let along = magnitude32(g0 -% (i32)*(q + h_ahead)) +% magnitude32(g0 -% (i32)*(q + h_behind))
# Branch-free choice: the smoother side is data.
let mask = -(isize)(across >= 2 *% along)
ahead = c_ahead ^ ((c_ahead ^ h_ahead) & mask)
behind = c_behind ^ ((c_behind ^ h_behind) & mask)
let a = q + ahead
let b = q + behind
let shift = (isize)f - 1
let value = (i32)*(a + shift) +% (i32)*(b + shift) +% 2 *% g0 -% (i32)*a -% (i32)*b
q[f] = clamp16((i64)half32(value))
## Red and blue for 2×2 blocks of green.
func xtrans_blocks(plane: const Plane*, rings: const Hexagons*, rgb: u16[], t: i64, l: i64, mrow: i64, mcol: i64):
let step = xtrans_side * xtrans_side * 3
let data = rgb.data
var offsets: isize[72]
for i: usize in 0..<72:
offsets[i] = (isize)rings.tile[i] * 3
for row in t + 2..<mrow - 2:
if (row - (i64)rings.solitary_row) % 3 == 0:
continue
var phase = (usize)((l + 2 - (i64)rings.solitary_col) % 3)
let row_base = (usize)(row % 3) * 3
var column_phase = (usize)((l + 2) % 3)
for col in l + 2..<mcol - 2:
let skip = phase == 0
let base = (row_base + column_phase) * 8
phase = 0 if phase == 2 else phase + 1
column_phase = 0 if column_phase == 2 else column_phase + 1
if skip:
continue
var q = data + (usize)tile_index(row, col, t, l)
var d: usize = 0
while d < 8:
let first = offsets[base + d]
let second = offsets[base + d + 1]
let a = q + first
let b = q + second
let g0 = (i32)q[1]
if first + second != 0:
let g = 3 *% g0 -% 2 *% (i32)a[1] -% (i32)b[1]
q[0] = clamp16((i64)third32(g +% 2 *% (i32)a[0] +% (i32)b[0]))
q[2] = clamp16((i64)third32(g +% 2 *% (i32)a[2] +% (i32)b[2]))
else:
let g = 2 *% g0 -% (i32)a[1] -% (i32)b[1]
q[0] = clamp16((i64)half32(g +% (i32)a[0] +% (i32)b[0]))
q[2] = clamp16((i64)half32(g +% (i32)a[2] +% (i32)b[2]))
d += 2
q = q + step
## ⌈2³² / n⌉ for n from 1 to 8.
let reciprocals: u64[9] = [0, 4294967296, 2147483648, 1431655766, 1073741824, 858993460, 715827883, 613566757, 536870912]
## Differentiate each direction in CIELAB, count homogeneous neighbours, and
## write the mean of the most homogeneous directions for [x0, x1) × [y0, y1).
## Each stage covers only the rows and columns the next reads, within the
## ranges LibRaw computes; what lies outside stays zero, as in LibRaw. The
## loops index the scratch with pointers: every offset stays within the
## `xtrans_side` square, whose three-pixel frame no stage reads past.
func xtrans_choose(plane: const Plane*, rings: const Hexagons*, table: const LabTable*, scratch: XtransScratch*, t: i64, l: i64, mr: i64, mc: i64, x0: usize, y0: usize, x1: usize, y1: usize, out: u16[], stride: usize, ox: usize, oy: usize):
let ts = xtrans_side
let area = xtrans_side * xtrans_side
let rgb = scratch.rgb.data
let lab = scratch.lab.data
let derivative = scratch.derivative.data
let steps: usize[4] = [1, ts, ts + 1, ts - 1]
let m0 = table.to_xyz[0]
let m1 = table.to_xyz[1]
let m2 = table.to_xyz[2]
let m3 = table.to_xyz[3]
let m4 = table.to_xyz[4]
let m5 = table.to_xyz[5]
let m6 = table.to_xyz[6]
let m7 = table.to_xyz[7]
let m8 = table.to_xyz[8]
let lightness = table.lightness.data
let red_green = table.red_green.data
let blue_yellow = table.blue_yellow.data
let r0 = (i64)y0 - t
let r1 = (i64)y1 - t
let c0 = (i64)x0 - l
let c1 = (i64)x1 - l
assert(r0 >= 4 and c0 >= 4 and r1 + 4 <= mr and c1 + 4 <= mc and mr <= (i64)ts and mc <= (i64)ts)
let lab_c0 = (usize)math.imax(2, c0 - 4)
let lab_c1 = (usize)math.imin(mc - 2, c1 + 4)
let drv_c0 = (usize)math.imax(3, c0 - 3)
let drv_c1 = (usize)math.imin(mc - 3, c1 + 3)
for d: usize in 0..<xtrans_directions:
let source = rgb + d * area * 3
for row in (usize)math.imax(2, r0 - 4)..<(usize)math.imin(mr - 2, r1 + 4):
for col in lab_c0..<lab_c1:
let o = (row *% ts +% col) *% 3
let p = source + o
let r = (i64)p[0]
let g = (i64)p[1]
let b = (i64)p[2]
let fx = clip16((m0 *% r +% m1 *% g +% m2 *% b +% 32768) >> 16)
let fy = clip16((m3 *% r +% m4 *% g +% m5 *% b +% 32768) >> 16)
let fz = clip16((m6 *% r +% m7 *% g +% m8 *% b +% 32768) >> 16)
let q = lab + o
q[0] = lightness[fy]
q[1] = red_green[fx] -% red_green[fy]
q[2] = blue_yellow[fy] -% blue_yellow[fz]
let f = steps[d & 3] * 3
let into = derivative + d * area
for row in (usize)math.imax(3, r0 - 3)..<(usize)math.imin(mr - 3, r1 + 3):
for col in drv_c0..<drv_c1:
let at = row *% ts +% col
let q = lab + at *% 3
let ahead = q + f
let behind = q - f
let g = 2 *% q[0] -% ahead[0] -% behind[0]
let a = 2 *% q[1] -% ahead[1] -% behind[1] +% divide40(g *% 500, 4739274258)
let b = 2 *% q[2] -% ahead[2] -% behind[2] -% divide40(g *% 500, 1895709704)
into[at] = (f32)((i64)g *% (i64)g +% (i64)a *% (i64)a +% (i64)b *% (i64)b)
xtrans_homogeneity(scratch, mr, mc, r0, r1, c0, c1)
xtrans_average(scratch, t, l, r0, r1, c0, c1, x0, y0, x1, y1, out, stride, ox, oy)
## Count, for each pixel and direction, the neighbours whose derivative is
## within eight times the pixel's smallest.
func xtrans_homogeneity(scratch: XtransScratch*, mr: i64, mc: i64, r0: i64, r1: i64, c0: i64, c1: i64):
let ts = xtrans_side
let area = xtrans_side * xtrans_side
let derivative = scratch.derivative.data
let homo = scratch.homogeneity.data
memory.set(scratch.homogeneity, 0)
let hom_c0 = (usize)math.imax(4, c0 - 2)
let hom_c1 = (usize)math.imin(mc - 4, c1 + 2)
for row in (usize)math.imax(4, r0 - 2)..<(usize)math.imin(mr - 4, r1 + 2):
for col in hom_c0..<hom_c1:
let at = row *% ts +% col
# The smallest derivative, compared by bit pattern (they are never
# negative, so bits order as values) without branches.
let bits = (const u32*)derivative
var smallest = (i64)bits[at]
for d: usize in 1..<xtrans_directions:
smallest = min64(smallest, (i64)bits[d *% area +% at])
let lowest = (u32)smallest
let threshold = *(const f32*)&lowest * 8.0f32
for d: usize in 0..<xtrans_directions:
let base = d *% area +% at
let p = derivative + base
let up = p - ts
let down = p + ts
# Counted without branches: which neighbours pass is data.
let count = (u8)(*(up - 1) <= threshold) +% (u8)(*up <= threshold) +% (u8)(*(up + 1) <= threshold) +% (u8)(*(p - 1) <= threshold) +% (u8)(*p <= threshold) +% (u8)(*(p + 1) <= threshold) +% (u8)(*(down - 1) <= threshold) +% (u8)(*down <= threshold) +% (u8)(*(down + 1) <= threshold)
homo[base] = count
## The mean of the most homogeneous directions over five-by-five windows:
## sliding five-wide sums of the homogeneity counts along the rows, sliding
## five-tall sums of those down the columns (into the spent derivative
## buffer), then per pixel the directions within an eighth of the best.
func xtrans_average(scratch: XtransScratch*, t: i64, l: i64, r0: i64, r1: i64, c0: i64, c1: i64, x0: usize, y0: usize, x1: usize, y1: usize, out: u16[], stride: usize, ox: usize, oy: usize):
let ts = xtrans_side
let area = xtrans_side * xtrans_side
let rgb = scratch.rgb.data
let homo = scratch.homogeneity.data
let sums = scratch.sums.data
let boxes = (u16*)scratch.derivative.data
let first = (usize)c0
let end = (usize)c1
for d: usize in 0..<xtrans_directions:
let base = d * area
for row in (usize)(r0 - 2)..<(usize)(r1 + 2):
let h = homo + (base + row * ts)
let o = sums + (base + row * ts)
var across = (u32)h[first - 2] +% (u32)h[first - 1] +% (u32)h[first] +% (u32)h[first + 1] +% (u32)h[first + 2]
o[first] = (u16)across
for col in first + 1..<end:
across = across +% (u32)h[col + 2] -% (u32)h[col - 3]
o[col] = (u16)across
let top = (usize)r0
let box_top = boxes + (base + top * ts)
let s0 = sums + (base + (top - 2) * ts)
let s1 = s0 + ts
let s2 = s1 + ts
let s3 = s2 + ts
let s4 = s3 + ts
for col in first..<end:
box_top[col] = s0[col] +% s1[col] +% s2[col] +% s3[col] +% s4[col]
for row in top + 1..<(usize)r1:
let above = boxes + (base + (row - 1) * ts)
let here = boxes + (base + row * ts)
let entering = sums + (base + (row + 2) * ts)
let leaving = sums + (base + (row - 3) * ts)
for col in first..<end:
here[col] = above[col] +% entering[col] -% leaving[col]
let target_base = out.data
for y in y0..<y1:
let row = (usize)((i64)y - t)
let target_row = target_base + ((oy + y - y0) * stride + ox) * 3
for x in x0..<x1:
let col = (usize)((i64)x - l)
let at = row *% ts +% col
let q = boxes + at
let h0 = (u32)*q
let h1 = (u32)*(q + 1 * area)
let h2 = (u32)*(q + 2 * area)
let h3 = (u32)*(q + 3 * area)
let h4 = (u32)*(q + 4 * area)
let h5 = (u32)*(q + 5 * area)
let h6 = (u32)*(q + 6 * area)
let h7 = (u32)*(q + 7 * area)
# Branch-free: of each opposite pair only the better survives
# (both on a tie), then the best of all.
let k0 = h0 *% (u32)(h0 >= h4)
let k4 = h4 *% (u32)(h4 >= h0)
let k1 = h1 *% (u32)(h1 >= h5)
let k5 = h5 *% (u32)(h5 >= h1)
let k2 = h2 *% (u32)(h2 >= h6)
let k6 = h6 *% (u32)(h6 >= h2)
let k3 = h3 *% (u32)(h3 >= h7)
let k7 = h7 *% (u32)(h7 >= h3)
var best = k0
best = best ^ ((best ^ k1) & -%(u32)(k1 > best))
best = best ^ ((best ^ k2) & -%(u32)(k2 > best))
best = best ^ ((best ^ k3) & -%(u32)(k3 > best))
best = best ^ ((best ^ k4) & -%(u32)(k4 > best))
best = best ^ ((best ^ k5) & -%(u32)(k5 > best))
best = best ^ ((best ^ k6) & -%(u32)(k6 > best))
best = best ^ ((best ^ k7) & -%(u32)(k7 > best))
best -= best >> 3
var red: u32 = 0
var green: u32 = 0
var blue: u32 = 0
var count: u32 = 0
let o = rgb + at *% 3
let take0 = (u32)(k0 >= best)
red +%= (u32)*o *% take0
green +%= (u32)*(o + 1) *% take0
blue +%= (u32)*(o + 2) *% take0
count +%= take0
let take1 = (u32)(k1 >= best)
red +%= (u32)*(o + 1 * area * 3) *% take1
green +%= (u32)*((o + 1 * area * 3) + 1) *% take1
blue +%= (u32)*((o + 1 * area * 3) + 2) *% take1
count +%= take1
let take2 = (u32)(k2 >= best)
red +%= (u32)*(o + 2 * area * 3) *% take2
green +%= (u32)*((o + 2 * area * 3) + 1) *% take2
blue +%= (u32)*((o + 2 * area * 3) + 2) *% take2
count +%= take2
let take3 = (u32)(k3 >= best)
red +%= (u32)*(o + 3 * area * 3) *% take3
green +%= (u32)*((o + 3 * area * 3) + 1) *% take3
blue +%= (u32)*((o + 3 * area * 3) + 2) *% take3
count +%= take3
let take4 = (u32)(k4 >= best)
red +%= (u32)*(o + 4 * area * 3) *% take4
green +%= (u32)*((o + 4 * area * 3) + 1) *% take4
blue +%= (u32)*((o + 4 * area * 3) + 2) *% take4
count +%= take4
let take5 = (u32)(k5 >= best)
red +%= (u32)*(o + 5 * area * 3) *% take5
green +%= (u32)*((o + 5 * area * 3) + 1) *% take5
blue +%= (u32)*((o + 5 * area * 3) + 2) *% take5
count +%= take5
let take6 = (u32)(k6 >= best)
red +%= (u32)*(o + 6 * area * 3) *% take6
green +%= (u32)*((o + 6 * area * 3) + 1) *% take6
blue +%= (u32)*((o + 6 * area * 3) + 2) *% take6
count +%= take6
let take7 = (u32)(k7 >= best)
red +%= (u32)*(o + 7 * area * 3) *% take7
green +%= (u32)*((o + 7 * area * 3) + 1) *% take7
blue +%= (u32)*((o + 7 * area * 3) + 2) *% take7
count +%= take7
# Division by the count (1 to 8) as a multiply by ⌈2³² / count⌉,
# exact for sums below 2²⁰.
let reciprocal = reciprocals[(usize)count]
let target = target_row + (x - x0) *% 3
target[0] = (u16)(((u64)red *% reciprocal) >> 32)
target[1] = (u16)(((u64)green *% reciprocal) >> 32)
target[2] = (u16)(((u64)blue *% reciprocal) >> 32)