dymokomi/luce-exr
Source of release 0.6.1
## OpenEXR v2 containers. All byte parsing and compression run in Luce Base.
import memory
import strings
from luce_std import math
from luce_raster import binary
from luce_compress import compress
import piz
import pxr24
from luce_raster.binary import Reader, Buffer
from luce_raster.model import Raster, Chunk, SampleType
from luce_raster.errors import corrupt, unsupported, invalid, limit
func int32(reader: Reader*) -> i64!:
return (i64)(i32)(try reader.uint(4))
func attribute_reader(image: const Raster*, name: str, kind: str) -> Reader!:
for item in image.attributes:
if item.name == name:
if item.type_name != kind:
error(corrupt, "EXR attribute has incorrect type")
return Reader(data = item.data)
error(corrupt, "required EXR attribute missing")
func header(image: Raster*, reader: Reader*, selected: bool = true) -> usize!:
var count: usize = 0
while true:
let name = try reader.cstring()
if name.length == 0:
break
let kind = try reader.cstring()
let length = (usize)(try reader.uint(4))
let data = try reader.take(length)
for item in image.attributes:
if item.name == name:
error(corrupt, "duplicate EXR attribute")
try image.attribute(name, kind, data)
count += 1
if count == 0:
error(corrupt, "empty EXR header")
let part_type = image.get("type")
if part_type == "deepscanline" or part_type == "deeptile":
error(unsupported, "deep EXR sample storage is not yet supported")
var window = try attribute_reader(image, "dataWindow", "box2i")
image.xmin = try int32(&window)
image.ymin = try int32(&window)
let xmax = try int32(&window)
let ymax = try int32(&window)
if xmax < image.xmin or ymax < image.ymin:
error(corrupt, "invalid EXR data window")
image.width = (usize)(xmax - image.xmin + 1)
image.height = (usize)(ymax - image.ymin + 1)
var compression = try attribute_reader(image, "compression", "compression")
image.compression = (u64)(try compression.byte())
if image.compression > 5:
error(unsupported, "EXR compression is not yet supported; use none, RLE, ZIPS, ZIP, PIZ or PXR24")
var channels = try attribute_reader(image, "channels", "chlist")
var previous: str = ""
while true:
let name = try channels.cstring()
if name.length == 0:
break
let kind = try channels.uint(4)
_ = try channels.take(4)
let xs = try channels.uint(4)
let ys = try channels.uint(4)
if xs != 1 or ys != 1:
error(unsupported, "EXR subsampled channels are not yet supported")
if kind > 2:
error(corrupt, "invalid EXR channel sample type")
if previous.length > 0 and name <= previous:
error(corrupt, "EXR channels are not sorted or unique")
try image.channel(name, .uint32 if kind == 0 else .float16 if kind == 1 else .float32)
previous = name
if selected:
_ = try image.validate()
elif image.width > 1000000 or image.height > 1000000 or image.channels.length == 0:
error(corrupt, "invalid unselected EXR part dimensions or channels")
var tw = image.width
var th: usize = 32 if image.compression == 4 else 16 if image.compression == 3 or image.compression == 5 else 1
let tiled = image.get("tiles").length != 0
image.interlaced = tiled
if tiled:
var tiles = try attribute_reader(image, "tiles", "tiledesc")
tw = (usize)(try tiles.uint(4))
th = (usize)(try tiles.uint(4))
let levels = try tiles.byte()
if levels != 0:
error(unsupported, "EXR mipmap/ripmap levels are not yet supported")
if tw == 0 or th == 0 or tw > 1000000 or th > 1000000:
error(corrupt, "invalid EXR tile dimensions")
let across = (image.width + tw - 1) // tw
let down = (image.height + th - 1) // th
let total = across * down
if total > image.max_bytes // memory.size_of(Chunk):
error(limit, "EXR chunk table exceeds byte limit")
if selected:
image.chunks = try new Chunk[total] in memory.heap
for i in 0..<total:
let x = i % across * tw
let y = i // across * th
image.chunks[i] = Chunk(x = x, y = y, width = image.width - x if image.width - x < tw else tw, height = image.height - y if image.height - y < th else th)
if image.get("chunkCount").length > 0:
var chunks = try attribute_reader(image, "chunkCount", "int")
if try chunks.uint(4) != (u64)total:
error(corrupt, "EXR chunkCount disagrees with dimensions")
return total
pub func probe(image: Raster*) -> !:
var reader = Reader(data = image.encoded)
if try reader.uint(4) != 20000630:
error(corrupt, "invalid EXR magic")
let version = try reader.uint(4)
if (version & 255) != 2 or version > 8191 or (version & 256) != 0:
error(unsupported, "unsupported EXR version or flags")
if (version & 2048) != 0:
error(unsupported, "deep EXR storage is not yet supported")
let multipart = (version & 4096) != 0
var before: usize = 0
var total: usize = 0
var parts: usize = 0
while true:
if parts >= 4096:
error(limit, "EXR exceeds part limit")
var size: usize = 0
if parts == image.frame:
size = try header(image, &reader)
else:
var ignored = Raster(max_bytes = image.max_bytes, max_pixels = image.max_pixels)
defer ignored.close()
size = try header(&ignored, &reader, selected = false)
if parts < image.frame:
before += size
total += size
parts += 1
if not multipart:
break
if reader.position >= reader.data.length:
error(corrupt, "truncated EXR multipart headers")
if reader.data[reader.position] == 0:
reader.position += 1
break
if image.frame >= parts:
error(invalid, "EXR part index out of range")
image.frames = parts
let offsets = try reader.take(total * 8)
var table = Reader(data = offsets, position = before * 8)
for i in 0..<image.chunks.length:
let offset = try table.uint(8)
var chunk = image.chunks[i]
var r = Reader(data = try binary.slice(image.encoded, offset, (u64)image.encoded.length - (offset if offset <= (u64)image.encoded.length else (u64)image.encoded.length)))
if multipart and try r.uint(4) != (u64)image.frame:
error(corrupt, "EXR chunk belongs to another part")
if image.interlaced:
let tx = try r.uint(4)
let ty = try r.uint(4)
let lx = try r.uint(4)
let ly = try r.uint(4)
var tiles = try attribute_reader(image, "tiles", "tiledesc")
let tw = try tiles.uint(4)
let th = try tiles.uint(4)
if tx * tw != (u64)chunk.x or ty * th != (u64)chunk.y or lx != 0 or ly != 0:
error(corrupt, "EXR tile offset table disagrees with chunk coordinates")
else:
let y = try int32(&r)
if y != image.ymin + (i64)chunk.y:
error(corrupt, "EXR scanline table disagrees with chunk coordinates")
chunk.size = try r.uint(4)
chunk.offset = offset + (u64)r.position
_ = try binary.slice(image.encoded, chunk.offset, chunk.size)
image.chunks[i] = chunk
image.file_format = "EXR"
func rle_decode(data: const u8[], out: u8[]) -> !:
var r = Reader(data = data)
var used: usize = 0
while r.position < data.length:
let code = (i32)(i8)(try r.byte())
let count = (usize)(-code if code < 0 else code + 1)
if count > out.length - used:
error(corrupt, "EXR RLE exceeds chunk")
if code < 0:
memory.copy(out[used..], try r.take(count), count)
else:
let value = try r.byte()
for i in 0..<count:
out[used + i] = value
used += count
if used != out.length:
error(corrupt, "incorrect EXR RLE chunk size")
pub func decode_chunk(image: Raster*, index: usize) -> !:
let chunk = image.chunks[index]
var stride: usize = 0
for channel in image.channels:
stride += 2 if channel.kind == .float16 else 4
let expected = chunk.width * chunk.height * stride
if expected > image.max_bytes:
error(limit, "EXR chunk exceeds byte limit")
let raw = try new u8[expected] --- in memory.heap
defer free(raw) in memory.heap
let data = try binary.slice(image.encoded, chunk.offset, chunk.size)
if data.length == expected:
memory.copy(raw, data, expected)
elif image.compression == 4:
try piz.decode(image, chunk, data, raw)
elif image.compression == 5:
try pxr24.decode(image, chunk, data, raw)
else:
if data.length > expected or image.compression == 0:
error(corrupt, "invalid EXR compressed chunk length")
let shuffled = try new u8[expected] --- in memory.heap
defer free(shuffled) in memory.heap
if image.compression == 1:
try rle_decode(data, shuffled)
else:
try compress.inflate(data, shuffled)
for i in 1..<expected:
shuffled[i] = shuffled[i - 1] +% shuffled[i] -% (u8)128
let half = (expected + 1) // 2
for i in 0..<expected:
raw[i] = shuffled[i // 2 + (half if i % 2 == 1 else 0)]
var r = Reader(data = raw)
for y in 0..<chunk.height:
for c in 0..<image.channels.length:
let kind = image.channels[c].kind
for x in 0..<chunk.width:
let value = try r.uint(2 if kind == .float16 else 4)
image.pixels[((chunk.y + y) * image.width + chunk.x + x) * image.channels.length + c] = binary.half((u16)value) if kind == .float16 else binary.float_value((u32)value) if kind == .float32 else (f64)value
func attribute(output: Buffer*, name: str, kind: str, data: const u8[]) -> !:
try output.text(name, true)
try output.text(kind, true)
try output.uint((u64)data.length, 4)
try output.put(data)
func rle_encode(data: const u8[], out: Buffer*) -> !:
var position: usize = 0
while position < data.length:
var count: usize = 1
while count < 128 and position + count < data.length and data[position + count] == data[position]:
count += 1
if count >= 3:
try out.byte((u8)(count - 1))
try out.byte(data[position])
position += count
else:
let start = position
position += count
while position - start < 127 and position < data.length:
if position + 2 < data.length and data[position] == data[position + 1] and data[position] == data[position + 2]:
break
position += 1
try out.byte((u8)(0 -% (i32)(position - start)))
try out.put(data[start..<position])
pub func encode(image: const Raster*, output: Buffer*, compression: str = "zip", tile: usize = 0) -> !:
let codec: u64 = 0 if compression == "none" else 1 if compression == "rle" else 2 if compression == "zips" else 3 if compression == "zip" else 4 if compression == "piz" else 255
if codec == 255:
error(unsupported, "EXR writer supports none, RLE, ZIPS, ZIP and PIZ")
if tile > 4096:
error(invalid, "EXR tile size exceeds 4096")
let nc = image.channels.length
let order = try new usize[nc] in memory.heap
defer free(order) in memory.heap
var long_names = false
for c in 0..<nc:
order[c] = c
if image.channels[c].name.length > 31:
long_names = true
if image.channels[c].kind != .uint32 and image.channels[c].kind != .float16 and image.channels[c].kind != .float32:
error(unsupported, "EXR requires UINT, HALF or FLOAT channels")
var i = c
while i > 0 and image.channels[order[i]].name < image.channels[order[i - 1]].name:
let value = order[i]
order[i] = order[i - 1]
order[i - 1] = value
i -= 1
for item in image.attributes:
if item.name.length > 31 or item.type_name.length > 31:
long_names = true
try output.uint(20000630, 4)
try output.uint(2 | ((u64)512 if tile > 0 else 0) | ((u64)1024 if long_names else 0), 4)
var data = Buffer()
defer data.close()
for c in order:
try data.text(image.channels[c].name, true)
try data.uint(0 if image.channels[c].kind == .uint32 else 1 if image.channels[c].kind == .float16 else 2, 4)
try data.uint(0, 4)
try data.uint(1, 4)
try data.uint(1, 4)
try data.byte(0)
try attribute(output, "channels", "chlist", data.view())
data.used = 0
try data.byte((u8)codec)
try attribute(output, "compression", "compression", data.view())
data.used = 0
try data.uint((u64)image.xmin, 4)
try data.uint((u64)image.ymin, 4)
try data.uint((u64)(image.xmin + (i64)image.width - 1), 4)
try data.uint((u64)(image.ymin + (i64)image.height - 1), 4)
try attribute(output, "dataWindow", "box2i", data.view())
let display = image.get("displayWindow")
try attribute(output, "displayWindow", "box2i", display.bytes if display.length == 16 else data.view())
try attribute(output, "lineOrder", "lineOrder", b"\0")
data.used = 0
try data.uint((u64)binary.float_bits(1.0), 4)
try attribute(output, "pixelAspectRatio", "float", data.view())
try attribute(output, "screenWindowWidth", "float", data.view())
try attribute(output, "screenWindowCenter", "v2f", b"\0\0\0\0\0\0\0\0")
if tile > 0:
data.used = 0
try data.uint((u64)tile, 4)
try data.uint((u64)tile, 4)
try data.byte(0)
try attribute(output, "tiles", "tiledesc", data.view())
for item in image.attributes:
if item.name == "channels" or item.name == "compression" or item.name == "dataWindow" or item.name == "displayWindow" or item.name == "lineOrder" or item.name == "pixelAspectRatio" or item.name == "screenWindowWidth" or item.name == "screenWindowCenter" or item.name == "tiles" or item.name == "chunkCount" or item.name == "type" or item.name == "version" or item.name == "name":
continue
try attribute(output, item.name, item.type_name, item.data)
try output.byte(0)
let tw = tile if tile > 0 else image.width
let th: usize = tile if tile > 0 else 32 if codec == 4 else 16 if codec == 3 else 1
let across = (image.width + tw - 1) // tw
let down = (image.height + th - 1) // th
let table = output.used
for i in 0..<across * down:
try output.uint(0, 8)
for index in 0..<across * down:
let x0 = index % across * tw
let y0 = index // across * th
let cols = image.width - x0 if image.width - x0 < tw else tw
let rows = image.height - y0 if image.height - y0 < th else th
var raw = Buffer(maximum = image.max_bytes)
defer raw.close()
for y in 0..<rows:
for c in order:
let kind = image.channels[c].kind
for x in 0..<cols:
let value = image.pixels[((y0 + y) * image.width + x0 + x) * nc + c]
var bits: u64 = 0
if kind == .uint32:
if not math.is_finite(value) or value < 0.0 or value > 4294967295.0 or math.floor(value) != value:
error(invalid, "EXR UINT sample out of range")
bits = (u64)value
else:
bits = (u64)binary.to_half(value) if kind == .float16 else (u64)binary.float_bits(value)
try raw.uint(bits, 2 if kind == .float16 else 4)
var packed = Buffer(maximum = image.max_bytes)
defer packed.close()
if codec == 4:
try piz.encode(image, Chunk(width = cols, height = rows), order, raw.view(), &packed)
elif codec > 0:
let shuffled = try new u8[raw.used] --- in memory.heap
defer free(shuffled) in memory.heap
let half = (raw.used + 1) // 2
for i in 0..<raw.used:
shuffled[i // 2 + (half if i % 2 == 1 else 0)] = raw.data[i]
var i = raw.used
while i > 1:
i -= 1
shuffled[i] = shuffled[i] -% shuffled[i - 1] +% (u8)128
if codec == 1:
try rle_encode(shuffled, &packed)
else:
let zipped = try compress.compress(shuffled)
defer free(zipped) in memory.heap
try packed.put(zipped)
let offset = output.used
for i in 0..<(usize)8:
output.data[table + index * 8 + i] = (u8)((u64)offset >> (i * 8))
if tile > 0:
try output.uint((u64)(index % across), 4)
try output.uint((u64)(index // across), 4)
try output.uint(0, 8)
else:
try output.uint((u64)(image.ymin + (i64)y0), 4)
let bytes = packed.view() if codec > 0 and packed.used < raw.used else raw.view()
try output.uint((u64)bytes.length, 4)
try output.put(bytes)