Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-exr

Source of release 0.6.2

0.6.2 / src / exr.lucb

## OpenEXR v2 containers. All byte parsing and compression run in Luce Base.
import memory
import strings
from luce_std import math
from luce_raster import binary
from luce_compress import compress
import piz
import pxr24
from luce_raster.binary import Reader, Buffer
from luce_raster.model import Raster, Chunk, SampleType
from luce_raster.errors import corrupt, unsupported, invalid, limit

func int32(reader: Reader*) -> i64!:
    return (i64)(i32)(try reader.uint(4))

func attribute_reader(image: const Raster*, name: str, kind: str) -> Reader!:
    for item in image.attributes:
        if item.name == name:
            if item.type_name != kind:
                error(corrupt, "EXR attribute has incorrect type")
            return Reader(data = item.data)
    error(corrupt, "required EXR attribute missing")

func header(image: Raster*, reader: Reader*, selected: bool = true) -> usize!:
    var count: usize = 0
    while true:
        let name = try reader.cstring()
        if name.length == 0:
            break
        let kind = try reader.cstring()
        let length = (usize)(try reader.uint(4))
        let data = try reader.take(length)
        for item in image.attributes:
            if item.name == name:
                error(corrupt, "duplicate EXR attribute")
        try image.attribute(name, kind, data)
        count += 1
    if count == 0:
        error(corrupt, "empty EXR header")
    let part_type = image.get("type")
    if part_type == "deepscanline" or part_type == "deeptile":
        error(unsupported, "deep EXR sample storage is not yet supported")
    var window = try attribute_reader(image, "dataWindow", "box2i")
    image.xmin = try int32(&window)
    image.ymin = try int32(&window)
    let xmax = try int32(&window)
    let ymax = try int32(&window)
    if xmax < image.xmin or ymax < image.ymin:
        error(corrupt, "invalid EXR data window")
    image.width = (usize)(xmax - image.xmin + 1)
    image.height = (usize)(ymax - image.ymin + 1)
    var compression = try attribute_reader(image, "compression", "compression")
    image.compression = (u64)(try compression.byte())
    if image.compression > 5:
        error(unsupported, "EXR compression is not yet supported; use none, RLE, ZIPS, ZIP, PIZ or PXR24")
    var channels = try attribute_reader(image, "channels", "chlist")
    var previous: str = ""
    while true:
        let name = try channels.cstring()
        if name.length == 0:
            break
        let kind = try channels.uint(4)
        _ = try channels.take(4)
        let xs = try channels.uint(4)
        let ys = try channels.uint(4)
        if xs != 1 or ys != 1:
            error(unsupported, "EXR subsampled channels are not yet supported")
        if kind > 2:
            error(corrupt, "invalid EXR channel sample type")
        if previous.length > 0 and name <= previous:
            error(corrupt, "EXR channels are not sorted or unique")
        try image.channel(name, .uint32 if kind == 0 else .float16 if kind == 1 else .float32)
        previous = name
    if selected:
        _ = try image.validate()
    elif image.width > 1000000 or image.height > 1000000 or image.channels.length == 0:
        error(corrupt, "invalid unselected EXR part dimensions or channels")
    var tw = image.width
    var th: usize = 32 if image.compression == 4 else 16 if image.compression == 3 or image.compression == 5 else 1
    let tiled = image.get("tiles").length != 0
    image.interlaced = tiled
    if tiled:
        var tiles = try attribute_reader(image, "tiles", "tiledesc")
        tw = (usize)(try tiles.uint(4))
        th = (usize)(try tiles.uint(4))
        let levels = try tiles.byte()
        if levels != 0:
            error(unsupported, "EXR mipmap/ripmap levels are not yet supported")
        if tw == 0 or th == 0 or tw > 1000000 or th > 1000000:
            error(corrupt, "invalid EXR tile dimensions")
    let across = (image.width + tw - 1) // tw
    let down = (image.height + th - 1) // th
    let total = across * down
    if total > image.max_bytes // memory.size_of(Chunk):
        error(limit, "EXR chunk table exceeds byte limit")
    if selected:
        image.chunks = try new Chunk[total] in memory.heap
        for i in 0..<total:
            let x = i % across * tw
            let y = i // across * th
            image.chunks[i] = Chunk(x = x, y = y, width = image.width - x if image.width - x < tw else tw, height = image.height - y if image.height - y < th else th)
    if image.get("chunkCount").length > 0:
        var chunks = try attribute_reader(image, "chunkCount", "int")
        if try chunks.uint(4) != (u64)total:
            error(corrupt, "EXR chunkCount disagrees with dimensions")
    return total

pub func probe(image: Raster*) -> !:
    var reader = Reader(data = image.encoded)
    if try reader.uint(4) != 20000630:
        error(corrupt, "invalid EXR magic")
    let version = try reader.uint(4)
    if (version & 255) != 2 or version > 8191 or (version & 256) != 0:
        error(unsupported, "unsupported EXR version or flags")
    if (version & 2048) != 0:
        error(unsupported, "deep EXR storage is not yet supported")
    let multipart = (version & 4096) != 0
    var before: usize = 0
    var total: usize = 0
    var parts: usize = 0
    while true:
        if parts >= 4096:
            error(limit, "EXR exceeds part limit")
        var size: usize = 0
        if parts == image.frame:
            size = try header(image, &reader)
        else:
            var ignored = Raster(max_bytes = image.max_bytes, max_pixels = image.max_pixels)
            defer ignored.close()
            size = try header(&ignored, &reader, selected = false)
        if parts < image.frame:
            before += size
        total += size
        parts += 1
        if not multipart:
            break
        if reader.position >= reader.data.length:
            error(corrupt, "truncated EXR multipart headers")
        if reader.data[reader.position] == 0:
            reader.position += 1
            break
    if image.frame >= parts:
        error(invalid, "EXR part index out of range")
    image.frames = parts
    let offsets = try reader.take(total * 8)
    var table = Reader(data = offsets, position = before * 8)
    for i in 0..<image.chunks.length:
        let offset = try table.uint(8)
        var chunk = image.chunks[i]
        var r = Reader(data = try binary.slice(image.encoded, offset, (u64)image.encoded.length - (offset if offset <= (u64)image.encoded.length else (u64)image.encoded.length)))
        if multipart and try r.uint(4) != (u64)image.frame:
            error(corrupt, "EXR chunk belongs to another part")
        if image.interlaced:
            let tx = try r.uint(4)
            let ty = try r.uint(4)
            let lx = try r.uint(4)
            let ly = try r.uint(4)
            var tiles = try attribute_reader(image, "tiles", "tiledesc")
            let tw = try tiles.uint(4)
            let th = try tiles.uint(4)
            if tx * tw != (u64)chunk.x or ty * th != (u64)chunk.y or lx != 0 or ly != 0:
                error(corrupt, "EXR tile offset table disagrees with chunk coordinates")
        else:
            let y = try int32(&r)
            if y != image.ymin + (i64)chunk.y:
                error(corrupt, "EXR scanline table disagrees with chunk coordinates")
        chunk.size = try r.uint(4)
        chunk.offset = offset + (u64)r.position
        _ = try binary.slice(image.encoded, chunk.offset, chunk.size)
        image.chunks[i] = chunk
    image.file_format = "EXR"

func rle_decode(data: const u8[], out: u8[]) -> !:
    var r = Reader(data = data)
    var used: usize = 0
    while r.position < data.length:
        let code = (i32)(i8)(try r.byte())
        let count = (usize)(-code if code < 0 else code + 1)
        if count > out.length - used:
            error(corrupt, "EXR RLE exceeds chunk")
        if code < 0:
            memory.copy(out[used..], try r.take(count), count)
        else:
            let value = try r.byte()
            for i in 0..<count:
                out[used + i] = value
        used += count
    if used != out.length:
        error(corrupt, "incorrect EXR RLE chunk size")

pub func decode_chunk(image: Raster*, index: usize) -> !:
    let chunk = image.chunks[index]
    var stride: usize = 0
    for channel in image.channels:
        stride += 2 if channel.kind == .float16 else 4
    let expected = chunk.width * chunk.height * stride
    if expected > image.max_bytes:
        error(limit, "EXR chunk exceeds byte limit")
    let raw = try new u8[expected] --- in memory.heap
    defer free(raw) in memory.heap
    let data = try binary.slice(image.encoded, chunk.offset, chunk.size)
    if data.length == expected:
        memory.copy(raw, data, expected)
    elif image.compression == 4:
        try piz.decode(image, chunk, data, raw)
    elif image.compression == 5:
        try pxr24.decode(image, chunk, data, raw)
    else:
        if data.length > expected or image.compression == 0:
            error(corrupt, "invalid EXR compressed chunk length")
        let shuffled = try new u8[expected] --- in memory.heap
        defer free(shuffled) in memory.heap
        if image.compression == 1:
            try rle_decode(data, shuffled)
        else:
            try compress.inflate(data, shuffled)
        for i in 1..<expected:
            shuffled[i] = shuffled[i - 1] +% shuffled[i] -% (u8)128
        let half = (expected + 1) // 2
        for i in 0..<expected:
            raw[i] = shuffled[i // 2 + (half if i % 2 == 1 else 0)]
    var r = Reader(data = raw)
    for y in 0..<chunk.height:
        for c in 0..<image.channels.length:
            let kind = image.channels[c].kind
            for x in 0..<chunk.width:
                let value = try r.uint(2 if kind == .float16 else 4)
                image.pixels[((chunk.y + y) * image.width + chunk.x + x) * image.channels.length + c] = binary.half((u16)value) if kind == .float16 else binary.float_value((u32)value) if kind == .float32 else (f64)value

func attribute(output: Buffer*, name: str, kind: str, data: const u8[]) -> !:
    try output.text(name, true)
    try output.text(kind, true)
    try output.uint((u64)data.length, 4)
    try output.put(data)

func rle_encode(data: const u8[], out: Buffer*) -> !:
    var position: usize = 0
    while position < data.length:
        var count: usize = 1
        while count < 128 and position + count < data.length and data[position + count] == data[position]:
            count += 1
        if count >= 3:
            try out.byte((u8)(count - 1))
            try out.byte(data[position])
            position += count
        else:
            let start = position
            position += count
            while position - start < 127 and position < data.length:
                if position + 2 < data.length and data[position] == data[position + 1] and data[position] == data[position + 2]:
                    break
                position += 1
            try out.byte((u8)(0 -% (i32)(position - start)))
            try out.put(data[start..<position])

pub func encode(image: const Raster*, output: Buffer*, compression: str = "zip", tile: usize = 0) -> !:
    let codec: u64 = 0 if compression == "none" else 1 if compression == "rle" else 2 if compression == "zips" else 3 if compression == "zip" else 4 if compression == "piz" else 255
    if codec == 255:
        error(unsupported, "EXR writer supports none, RLE, ZIPS, ZIP and PIZ")
    if tile > 4096:
        error(invalid, "EXR tile size exceeds 4096")
    let nc = image.channels.length
    let order = try new usize[nc] in memory.heap
    defer free(order) in memory.heap
    var long_names = false
    for c in 0..<nc:
        order[c] = c
        if image.channels[c].name.length > 31:
            long_names = true
        if image.channels[c].kind != .uint32 and image.channels[c].kind != .float16 and image.channels[c].kind != .float32:
            error(unsupported, "EXR requires UINT, HALF or FLOAT channels")
        var i = c
        while i > 0 and image.channels[order[i]].name < image.channels[order[i - 1]].name:
            let value = order[i]
            order[i] = order[i - 1]
            order[i - 1] = value
            i -= 1
    for item in image.attributes:
        if item.name.length > 31 or item.type_name.length > 31:
            long_names = true
    try output.uint(20000630, 4)
    try output.uint(2 | ((u64)512 if tile > 0 else 0) | ((u64)1024 if long_names else 0), 4)
    var data = Buffer()
    defer data.close()
    for c in order:
        try data.text(image.channels[c].name, true)
        try data.uint(0 if image.channels[c].kind == .uint32 else 1 if image.channels[c].kind == .float16 else 2, 4)
        try data.uint(0, 4)
        try data.uint(1, 4)
        try data.uint(1, 4)
    try data.byte(0)
    try attribute(output, "channels", "chlist", data.view())
    data.used = 0
    try data.byte((u8)codec)
    try attribute(output, "compression", "compression", data.view())
    data.used = 0
    try data.uint((u64)image.xmin, 4)
    try data.uint((u64)image.ymin, 4)
    try data.uint((u64)(image.xmin + (i64)image.width - 1), 4)
    try data.uint((u64)(image.ymin + (i64)image.height - 1), 4)
    try attribute(output, "dataWindow", "box2i", data.view())
    let display = image.get("displayWindow")
    try attribute(output, "displayWindow", "box2i", display.bytes if display.length == 16 else data.view())
    try attribute(output, "lineOrder", "lineOrder", b"\0")
    data.used = 0
    try data.uint((u64)binary.float_bits(1.0), 4)
    try attribute(output, "pixelAspectRatio", "float", data.view())
    try attribute(output, "screenWindowWidth", "float", data.view())
    try attribute(output, "screenWindowCenter", "v2f", b"\0\0\0\0\0\0\0\0")
    if tile > 0:
        data.used = 0
        try data.uint((u64)tile, 4)
        try data.uint((u64)tile, 4)
        try data.byte(0)
        try attribute(output, "tiles", "tiledesc", data.view())
    for item in image.attributes:
        if item.name == "channels" or item.name == "compression" or item.name == "dataWindow" or item.name == "displayWindow" or item.name == "lineOrder" or item.name == "pixelAspectRatio" or item.name == "screenWindowWidth" or item.name == "screenWindowCenter" or item.name == "tiles" or item.name == "chunkCount" or item.name == "type" or item.name == "version" or item.name == "name":
            continue
        try attribute(output, item.name, item.type_name, item.data)
    try output.byte(0)
    let tw = tile if tile > 0 else image.width
    let th: usize = tile if tile > 0 else 32 if codec == 4 else 16 if codec == 3 else 1
    let across = (image.width + tw - 1) // tw
    let down = (image.height + th - 1) // th
    let table = output.used
    for i in 0..<across * down:
        try output.uint(0, 8)
    for index in 0..<across * down:
        let x0 = index % across * tw
        let y0 = index // across * th
        let cols = image.width - x0 if image.width - x0 < tw else tw
        let rows = image.height - y0 if image.height - y0 < th else th
        var raw = Buffer(maximum = image.max_bytes)
        defer raw.close()
        for y in 0..<rows:
            for c in order:
                let kind = image.channels[c].kind
                for x in 0..<cols:
                    let value = image.pixels[((y0 + y) * image.width + x0 + x) * nc + c]
                    var bits: u64 = 0
                    if kind == .uint32:
                        if not math.is_finite(value) or value < 0.0 or value > 4294967295.0 or math.floor(value) != value:
                            error(invalid, "EXR UINT sample out of range")
                        bits = (u64)value
                    else:
                        bits = (u64)binary.to_half(value) if kind == .float16 else (u64)binary.float_bits(value)
                    try raw.uint(bits, 2 if kind == .float16 else 4)
        var packed = Buffer(maximum = image.max_bytes)
        defer packed.close()
        if codec == 4:
            try piz.encode(image, Chunk(width = cols, height = rows), order, raw.view(), &packed)
        elif codec > 0:
            let shuffled = try new u8[raw.used] --- in memory.heap
            defer free(shuffled) in memory.heap
            let half = (raw.used + 1) // 2
            for i in 0..<raw.used:
                shuffled[i // 2 + (half if i % 2 == 1 else 0)] = raw.data[i]
            var i = raw.used
            while i > 1:
                i -= 1
                shuffled[i] = shuffled[i] -% shuffled[i - 1] +% (u8)128
            if codec == 1:
                try rle_encode(shuffled, &packed)
            else:
                let zipped = try compress.compress(shuffled)
                defer free(zipped) in memory.heap
                try packed.put(zipped)
        let offset = output.used
        for i in 0..<(usize)8:
            output.data[table + index * 8 + i] = (u8)((u64)offset >> (i * 8))
        if tile > 0:
            try output.uint((u64)(index % across), 4)
            try output.uint((u64)(index // across), 4)
            try output.uint(0, 8)
        else:
            try output.uint((u64)(image.ymin + (i64)y0), 4)
        let bytes = packed.view() if codec > 0 and packed.used < raw.used else raw.view()
        try output.uint((u64)bytes.length, 4)
        try output.put(bytes)