Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-tiff

Source of release 0.5.1

0.5.1 / src / tiff.lucb

## TIFF 6.0/BigTIFF directories, strips/tiles, separate planes and predictors.
import memory
from luce_std import math
from luce_raster import binary
from luce_compress import compress
from luce_raster.binary import Reader, Buffer
from luce_raster.model import Raster, Chunk, SampleType
from luce_raster.errors import corrupt, unsupported, invalid, limit

struct Field:
    var data: const u8[]
    var count: usize
    var width: usize
    var little: bool = true

    func at(index: usize, fallback: u64 = 0) -> u64!:
        if self.count == 0:
            return fallback
        if index >= self.count or self.width == 0 or self.width > 8:
            error(corrupt, "TIFF field index or type invalid")
        var reader = Reader(data = self.data, position = index * self.width, little = self.little)
        return try reader.uint(self.width)

func field(image: const Raster*, directory: u64, big: bool, tag: u64) -> Field!:
    var r = Reader(data = try binary.slice(image.encoded, directory, (u64)image.encoded.length - directory), little = image.little)
    let count = try r.uint(8 if big else 2)
    if count > 65536:
        error(limit, "TIFF directory has too many entries")
    for i in 0..<(usize)count:
        let name = try r.uint(2)
        let kind = try r.uint(2)
        let items = try r.uint(8 if big else 4)
        let inline_bytes = try r.take(8 if big else 4)
        if name == tag:
            let widths: usize[19] = [0, 1, 1, 2, 4, 8, 1, 1, 2, 4, 8, 4, 8, 4, 0, 0, 8, 8, 8]
            if kind >= 19 or widths[(usize)kind] == 0:
                error(unsupported, "unsupported TIFF field type")
            let width = widths[(usize)kind]
            let bytes = items *? (u64)width else error(corrupt, "TIFF field extent overflows")
            var data = inline_bytes[..<0]
            if bytes <= (u64)inline_bytes.length:
                data = inline_bytes[..<(usize)bytes]
            else:
                var offset_reader = Reader(data = inline_bytes, little = image.little)
                data = try binary.slice(image.encoded, try offset_reader.uint(inline_bytes.length), bytes)
            return Field(data, (usize)items, width, image.little)
    return Field()

func scalar(image: const Raster*, directory: u64, big: bool, tag: u64, fallback: u64) -> u64!:
    return try (try field(image, directory, big, tag)).at(0, fallback)

pub func probe(image: Raster*) -> !:
    var r = Reader(data = image.encoded)
    let order = (str)(try r.take(2))
    if order != "II" and order != "MM":
        error(corrupt, "invalid TIFF byte order")
    image.little = order == "II"
    r.little = image.little
    let magic = try r.uint(2)
    let big = magic == 43
    if magic != 42 and not big:
        error(corrupt, "invalid TIFF signature")
    if big and ((try r.uint(2)) != 8 or (try r.uint(2)) != 0):
        error(corrupt, "invalid BigTIFF header")
    var offset = try r.uint(8 if big else 4)
    var chosen: u64 = 0
    var visited: u64[4096]
    var count: usize = 0
    while offset != 0:
        if count == visited.length:
            error(limit, "TIFF exceeds directory limit")
        for i in 0..<count:
            if visited[i] == offset:
                error(corrupt, "cyclic TIFF directory chain")
        visited[count] = offset
        if count == image.frame:
            chosen = offset
        count += 1
        var directory = Reader(data = try binary.slice(image.encoded, offset, (u64)image.encoded.length - (offset if offset <= (u64)image.encoded.length else (u64)image.encoded.length)), little = image.little)
        let entries = try directory.uint(8 if big else 2)
        if entries > 65536:
            error(limit, "TIFF directory exceeds entry limit")
        _ = try directory.take((usize)entries * (20 if big else 12))
        offset = try directory.uint(8 if big else 4)
    if chosen == 0:
        error(invalid, "TIFF frame out of range")
    image.frames = count
    let w = (usize)(try scalar(image, chosen, big, 256, 0))
    let h = (usize)(try scalar(image, chosen, big, 257, 0))
    image.width = w
    image.height = h
    let samples = (usize)(try scalar(image, chosen, big, 277, 1))
    if samples < 1 or samples > 4:
        error(unsupported, "TIFF currently supports one to four samples per pixel")
    let depths = try field(image, chosen, big, 258)
    image.bits = (usize)(try depths.at(0, 1))
    for i in 0..<depths.count:
        if try depths.at(i) != (u64)image.bits:
            error(unsupported, "mixed TIFF bit depths")
    image.photo = try scalar(image, chosen, big, 262, 1)
    image.compression = try scalar(image, chosen, big, 259, 1)
    image.planar = try scalar(image, chosen, big, 284, 1)
    image.predictor = try scalar(image, chosen, big, 317, 1)
    image.orientation = try scalar(image, chosen, big, 274, 1)
    let sample_kind = try scalar(image, chosen, big, 339, 1)
    let bits = image.bits
    var kind = SampleType.uint8
    if sample_kind == 1:
        if bits != 1 and bits != 2 and bits != 4 and bits != 8 and bits != 16 and bits != 32:
            error(unsupported, "unsupported TIFF integer bit depth")
        kind = .uint8 if bits <= 8 else .uint16 if bits == 16 else .uint32
    elif sample_kind == 2 and (bits == 8 or bits == 16 or bits == 32):
        kind = .int8 if bits == 8 else .int16 if bits == 16 else .int32
    elif sample_kind == 3 and (bits == 16 or bits == 32 or bits == 64):
        kind = .float16 if bits == 16 else .float32 if bits == 32 else .float64
    else:
        error(unsupported, "unsupported TIFF sample type")
    if image.photo > 3 or image.planar < 1 or image.planar > 2 or image.orientation < 1 or image.orientation > 8 or (image.photo == 2 and samples < 3):
        error(unsupported, "unsupported TIFF color or storage layout")
    if image.predictor < 1 or image.predictor > 3 or (image.predictor == 2 and sample_kind == 3) or (image.predictor == 3 and sample_kind != 3):
        error(unsupported, "unsupported TIFF predictor/sample combination")
    if image.compression != 1 and image.compression != 5 and image.compression != 8 and image.compression != 32946 and image.compression != 32773:
        error(unsupported, "unsupported TIFF compression")
    if (try scalar(image, chosen, big, 266, 1)) != 1:
        error(unsupported, "reversed TIFF fill order is not supported")
    if image.photo == 3:
        if samples != 1 or bits > 16 or sample_kind != 1:
            error(corrupt, "invalid TIFF palette layout")
        let palette = try field(image, chosen, big, 320)
        if palette.width != 2 or palette.count != ((usize)1 << bits) * 3:
            error(corrupt, "invalid TIFF color map")
        image.palette = palette.data
        try image.standard(3, .uint16)
    else:
        try image.standard(samples, kind)
    if image.orientation >= 5:
        image.width = h
        image.height = w
    _ = try image.validate()
    let tiled = (try scalar(image, chosen, big, 322, 0)) != 0
    let tw = (usize)(try scalar(image, chosen, big, 322, (u64)w))
    var th = (usize)(try scalar(image, chosen, big, 323 if tiled else 278, (u64)h))
    if not tiled and th > h:
        th = h
    if tw == 0 or th == 0 or tw > 1000000 or th > 1000000:
        error(corrupt, "invalid TIFF chunk dimensions")
    let across = (w + tw - 1) // tw
    let down = (h + th - 1) // th
    let planes = samples if image.planar == 2 else 1
    let total = across * down * planes
    let offsets = try field(image, chosen, big, 324 if tiled else 273)
    let sizes = try field(image, chosen, big, 325 if tiled else 279)
    if offsets.count != total or sizes.count != total or total > image.max_bytes // memory.size_of(Chunk):
        error(corrupt, "invalid TIFF chunk table")
    image.chunks = try new Chunk[total] in memory.heap
    for index in 0..<total:
        let start = try offsets.at(index)
        let size = try sizes.at(index)
        _ = try binary.slice(image.encoded, start, size)
        let local_index = index % (across * down)
        let y = local_index // across * th
        image.chunks[index] = Chunk(start, size, local_index % across * tw, y, tw, th if tiled else (h - y if h - y < th else th), index // (across * down))
    let description = try field(image, chosen, big, 270)
    if description.count > 0:
        let data = description.data
        try image.attribute("description", "string", data[..<data.length - 1] if data[data.length - 1] == 0 else data)
    image.file_format = "TIFF"

func packbits(data: const u8[], out: u8[]) -> !:
    var r = Reader(data = data)
    var used: usize = 0
    while r.position < data.length:
        let code = try r.byte()
        if code <= 127:
            let count = (usize)code + 1
            if count > out.length - used:
                error(corrupt, "PackBits literal exceeds chunk")
            memory.copy(out[used..], try r.take(count), count)
            used += count
        elif code > 128:
            let count = 257 - (usize)code
            if count > out.length - used:
                error(corrupt, "PackBits run exceeds chunk")
            let value = try r.byte()
            for i in 0..<count:
                out[used] = value
                used += 1
    if used != out.length:
        error(corrupt, "PackBits chunk has incorrect length")

func lzw(data: const u8[], out: u8[]) -> !:
    var prefix: u16[4096]
    var suffix: u8[4096]
    var stack: u8[4096]
    var bit: usize = 0
    var width: usize = 9
    var next: usize = 258
    var previous: usize = 4096
    var first: u8 = 0
    var used: usize = 0
    while true:
        if bit + width > data.length * 8:
            error(corrupt, "truncated TIFF LZW")
        var code: usize = 0
        for i in 0..<width:
            code = (code << 1) | (usize)((data[bit // 8] >> (7 - bit % 8)) & 1)
            bit += 1
        if code == 256:
            width = 9
            next = 258
            previous = 4096
            continue
        if code == 257:
            break
        let original = code
        var count: usize = 0
        if code == next and previous != 4096:
            stack[count] = first
            count += 1
            code = previous
        elif code >= next:
            error(corrupt, "invalid TIFF LZW code")
        while code >= 258:
            if code >= next or count == stack.length:
                error(corrupt, "invalid TIFF LZW dictionary chain")
            stack[count] = suffix[code]
            count += 1
            code = (usize)prefix[code]
        if code > 255 or count == stack.length:
            error(corrupt, "invalid TIFF LZW literal")
        first = (u8)code
        stack[count] = first
        count += 1
        if count > out.length - used:
            error(corrupt, "TIFF LZW exceeds chunk")
        while count > 0:
            count -= 1
            out[used] = stack[count]
            used += 1
        if previous != 4096 and next < 4096:
            prefix[next] = (u16)previous
            suffix[next] = first
            next += 1
            if next == ((usize)1 << width) - 1 and width < 12:
                width += 1
        previous = original
    if used != out.length:
        error(corrupt, "TIFF LZW chunk has incorrect length")

func number(value: u64, kind: SampleType) -> f64:
    if kind == .float16:
        return binary.half((u16)value)
    if kind == .float32:
        return binary.float_value((u32)value)
    if kind == .float64:
        return binary.double_value(value)
    if kind == .int8:
        return (f64)(i8)value
    if kind == .int16:
        return (f64)(i16)value
    if kind == .int32:
        return (f64)(i32)value
    return (f64)value

pub func decode_chunk(image: Raster*, index: usize) -> !:
    let chunk = image.chunks[index]
    let nc: usize = 1 if image.photo == 3 else image.channels.length
    let stride: usize = 1 if image.planar == 2 else nc
    let rowbytes = (chunk.width * stride * image.bits + 7) // 8
    if chunk.height > image.max_bytes // rowbytes:
        error(limit, "TIFF decoded chunk exceeds byte limit")
    let raw = try new u8[rowbytes * chunk.height] --- in memory.heap
    defer free(raw) in memory.heap
    let data = try binary.slice(image.encoded, chunk.offset, chunk.size)
    if image.compression == 1:
        if data.length != raw.length:
            error(corrupt, "uncompressed TIFF chunk length mismatch")
        memory.copy(raw, data, raw.length)
    elif image.compression == 5:
        try lzw(data, raw)
    elif image.compression == 32773:
        try packbits(data, raw)
    else:
        try compress.inflate(data, raw)
    let source_width = image.height if image.orientation >= 5 else image.width
    let source_height = image.width if image.orientation >= 5 else image.height
    let cols = source_width - chunk.x if source_width - chunk.x < chunk.width else chunk.width
    let rows = source_height - chunk.y if source_height - chunk.y < chunk.height else chunk.height
    let kind = image.channels[0].kind
    let mask: u64 = 18446744073709551615 if image.bits == 64 else ((u64)1 << image.bits) - 1
    let temporary = try new u8[rowbytes if image.predictor == 3 else 0] --- in memory.heap
    defer free(temporary) in memory.heap
    for y in 0..<rows:
        let row = raw[y * rowbytes..<(y + 1) * rowbytes]
        if image.predictor == 3:
            for i in stride..<rowbytes:
                row[i] +%= row[i - stride]
            let samples = chunk.width * stride
            let bytes = image.bits // 8
            for i in 0..<samples:
                for b in 0..<bytes:
                    temporary[i * bytes + (bytes - 1 - b if image.little else b)] = row[b * samples + i]
            memory.copy(row, temporary, rowbytes)
        var previous: u64[4]
        for x in 0..<cols:
            let sx = chunk.x + x
            let sy = chunk.y + y
            var dx = sx
            var dy = sy
            let o = image.orientation
            if o == 2 or o == 3:
                dx = source_width - 1 - sx
            if o == 3 or o == 4:
                dy = source_height - 1 - sy
            if o >= 5:
                dx = source_height - 1 - sy if o == 6 or o == 7 else sy
                dy = source_width - 1 - sx if o == 7 or o == 8 else sx
            let target = (dy * image.width + dx) * image.channels.length
            for c in 0..<stride:
                let sample_index = x * stride + c
                var value: u64 = 0
                if image.bits < 8:
                    value = ((u64)row[sample_index * image.bits // 8] >> (8 - image.bits - sample_index * image.bits % 8)) & mask
                else:
                    var reader = Reader(data = row, position = sample_index * (image.bits // 8), little = image.little)
                    value = try reader.uint(image.bits // 8)
                if image.predictor == 2:
                    value = (value +% previous[c]) & mask
                    previous[c] = value
                if image.photo == 3:
                    for color in 0..<(usize)3:
                        var palette = Reader(data = image.palette, position = (color * ((usize)1 << image.bits) + (usize)value) * 2, little = image.little)
                        image.pixels[target + color] = (f64)(try palette.uint(2))
                else:
                    let channel = chunk.plane if image.planar == 2 else c
                    var result = number(value, kind)
                    if image.bits < 8:
                        result = result * 255.0 / (f64)mask
                    if image.photo == 0 and channel == 0:
                        result = (255.0 if image.bits <= 8 else (f64)mask if (i64)kind <= 3 else 1.0) - result
                    image.pixels[target + channel] = result

func entry(output: Buffer*, tag: u64, kind: u64, count: u64, value: u64, big: bool) -> !:
    try output.uint(tag, 2)
    try output.uint(kind, 2)
    try output.uint(count, 8 if big else 4)
    try output.uint(value, 8 if big else 4)

## Deterministic strip/tile writer. Compression is none or pure-Base DEFLATE.
pub func encode(image: const Raster*, output: Buffer*, compression: str = "zip", tile: usize = 0, big: bool = false) -> !:
    let nc = image.channels.length
    let kind = image.channels[0].kind
    if nc > 4 or (compression != "zip" and compression != "deflate" and compression != "none"):
        error(unsupported, "TIFF writer supports 1-4 channels and none/deflate compression")
    for channel in image.channels:
        if channel.kind != kind:
            error(unsupported, "TIFF requires matching channel types")
    if tile != 0 and (tile < 16 or tile > 4096 or tile % 16 != 0):
        error(invalid, "TIFF tile size must be a multiple of 16 in 16-4096")
    let bits: usize = 8 if kind == .uint8 or kind == .int8 else 16 if kind == .uint16 or kind == .float16 or kind == .int16 else 64 if kind == .float64 else 32
    let sample_kind: u64 = 3 if kind == .float16 or kind == .float32 or kind == .float64 else 2 if (i64)kind >= 7 else 1
    let tw = tile if tile > 0 else image.width
    let th = tile if tile > 0 else (64 if image.height > 64 else image.height)
    let across = (image.width + tw - 1) // tw
    let down = (image.height + th - 1) // th
    let chunks = across * down
    let offsets = try new u64[chunks] in memory.heap
    defer free(offsets) in memory.heap
    let sizes = try new u64[chunks] in memory.heap
    defer free(sizes) in memory.heap
    try output.text("II")
    try output.uint(43 if big else 42, 2)
    if big:
        try output.uint(8, 2)
        try output.uint(0, 2)
    let directory_pointer = output.used
    try output.uint(0, 8 if big else 4)
    for index in 0..<chunks:
        let x0 = index % across * tw
        let y0 = index // across * th
        let rows = th if tile > 0 else (image.height - y0 if image.height - y0 < th else th)
        var raw = Buffer(maximum = image.max_bytes)
        defer raw.close()
        for y in 0..<rows:
            for x in 0..<tw:
                for c in 0..<nc:
                    let value = image.pixels[((y0 + y) * image.width + x0 + x) * nc + c] if x0 + x < image.width and y0 + y < image.height else 0.0
                    var encoded: u64 = 0
                    if kind == .float16:
                        encoded = (u64)binary.to_half(value)
                    elif kind == .float32:
                        encoded = (u64)binary.float_bits(value)
                    elif kind == .float64:
                        encoded = binary.double_bits(value)
                    else:
                        let lo = -(f64)((u64)1 << (bits - 1)) if sample_kind == 2 else 0.0
                        let hi = (f64)(((u64)1 << (bits - 1)) - 1) if sample_kind == 2 else (f64)(((u64)1 << bits) - 1)
                        if not math.is_finite(value) or value < lo or value > hi or math.floor(value) != value:
                            error(invalid, "TIFF integer sample out of range")
                        encoded = (u64)(i64)value if sample_kind == 2 else (u64)value
                    try raw.uint(encoded, bits // 8)
        offsets[index] = (u64)output.used
        if compression == "none":
            try output.put(raw.view())
        else:
            let packed = try compress.compress(raw.view())
            defer free(packed) in memory.heap
            try output.put(packed)
        sizes[index] = (u64)output.used - offsets[index]
    let slot: usize = 8 if big else 4
    let offsets_at = (u64)output.used
    for value in offsets:
        try output.uint(value, slot)
    let sizes_at = (u64)output.used
    for value in sizes:
        try output.uint(value, slot)
    let depths_at = (u64)output.used
    for c in 0..<nc:
        try output.uint((u64)bits, 2)
    var depth_value = depths_at
    if nc * 2 <= slot:
        depth_value = 0
        for c in 0..<nc:
            depth_value |= (u64)bits << (16 * c)
    let directory = (u64)output.used
    for i in 0..<slot:
        output.data[directory_pointer + i] = (u8)(directory >> (8 * i))
    let entries: u64 = 13 + (1 if nc == 2 or nc == 4 else 0) + (1 if tile > 0 else 0)
    try output.uint(entries, 8 if big else 2)
    try entry(output, 256, 4, 1, (u64)image.width, big)
    try entry(output, 257, 4, 1, (u64)image.height, big)
    try entry(output, 258, 3, (u64)nc, depth_value, big)
    try entry(output, 259, 3, 1, 1 if compression == "none" else 8, big)
    try entry(output, 262, 3, 1, 2 if nc >= 3 else 1, big)
    if tile == 0:
        try entry(output, 273, 16 if big else 4, (u64)chunks, offsets[0] if chunks == 1 else offsets_at, big)
    try entry(output, 274, 3, 1, 1, big)
    try entry(output, 277, 3, 1, (u64)nc, big)
    if tile == 0:
        try entry(output, 278, 4, 1, (u64)th, big)
        try entry(output, 279, 16 if big else 4, (u64)chunks, sizes[0] if chunks == 1 else sizes_at, big)
    try entry(output, 284, 3, 1, 1, big)
    try entry(output, 317, 3, 1, 1, big)
    if tile > 0:
        try entry(output, 322, 4, 1, (u64)tw, big)
        try entry(output, 323, 4, 1, (u64)th, big)
        try entry(output, 324, 16 if big else 4, (u64)chunks, offsets[0] if chunks == 1 else offsets_at, big)
        try entry(output, 325, 16 if big else 4, (u64)chunks, sizes[0] if chunks == 1 else sizes_at, big)
    if nc == 2 or nc == 4:
        try entry(output, 338, 3, 1, 2, big)
    try entry(output, 339, 3, 1, sample_kind, big)
    try output.uint(0, slot)

# mark: Rows of chunks ========================================================================

## Receives a TIFF's rows `top..<top + part.height` decoded into `part`, a raster
## of just those rows with the source's channels. Valid only during the call.
pub type RowsSink = func(void*?, usize, const Raster*) -> unit!

## Whether a probed TIFF decodes a row of strips or tiles at a time: upright,
## its chunks in rows from the top (for each plane).
pub func streams_by_rows(image: const Raster*) -> bool:
    if image.orientation != 1 or image.chunks.length == 0:
        return false
    let planes = image.channels.length if image.planar == 2 else 1
    if image.chunks.length % planes != 0:
        return false
    let per_plane = image.chunks.length // planes
    for index in 1..<image.chunks.length:
        if index % per_plane != 0 and image.chunks[index].y < image.chunks[index - 1].y:
            return false
    return true

## A probed TIFF decoded a row of strips or tiles at a time (`streams_by_rows`):
## each row of chunks into a raster of just their rows, handed to `sink`, so the
## whole picture is never held. False, with nothing decoded, for any other TIFF.
pub func decode_rows(image: Raster*, sink: RowsSink, context: void*?) -> bool!:
    if not streams_by_rows(image):
        return false
    let planes = image.channels.length if image.planar == 2 else 1
    let per_plane = image.chunks.length // planes
    var first: usize = 0
    while first < per_plane:
        var last = first + 1
        while last < per_plane and image.chunks[last].y == image.chunks[first].y:
            last += 1
        try decode_chunk_row(image, first, last, per_plane, sink, context)
        first = last
    return true

## Chunks `first..<last` of each plane (one row of them) decoded into a raster
## of just their rows, the chunks read as if at its top, and handed to `sink`.
func decode_chunk_row(image: Raster*, first: usize, last: usize, per_plane: usize, sink: RowsSink, context: void*?) -> !:
    let top = image.chunks[first].y
    let rows = image.chunks[first].height if image.chunks[first].height < image.height - top else image.height - top
    var part = *image
    part.height = rows
    part.pixels = try new f64[image.width * rows * image.channels.length] --- in memory.heap
    defer free(part.pixels) in memory.heap
    for index in 0..<part.pixels.length:
        part.pixels[index] = 0.0
    var one: Chunk[1] = ---
    part.chunks = one[..<1]
    for plane in 0..<image.chunks.length // per_plane:
        for index in first..<last:
            one[0] = image.chunks[plane * per_plane + index]
            one[0].y = 0
            try decode_chunk(&part, 0)
    try sink(context, top, &part)

func row_total(context: void*?, top: usize, part: const Raster*) -> !:
    let seen = (f64*)(context else trap("a test sink lost its total"))
    for y in 0..<part.height:
        for x in 0..<part.width:
            *seen += part.pixels[(y * part.width + x) * part.channels.length] * (f64)(top + y + 1)

test "a stripped TIFF decodes a row of strips at a time, as a whole decode does":
    var image = Raster(width = 40, height = 150)
    defer image.close()
    try image.standard(1, SampleType.uint8)
    image.pixels = try new f64[40 * 150] --- in memory.heap
    for index in 0..<image.pixels.length:
        image.pixels[index] = (f64)((index * 7) % 251)
    var output = Buffer()
    defer output.close()
    try encode(&image, &output, "zip")
    var read = Raster()
    defer read.close()
    read.encoded = try new u8[output.used] --- in memory.heap
    memory.copy(read.encoded, output.view(), output.used)
    try probe(&read)
    var total = 0.0
    assert(try decode_rows(&read, row_total, (void*)&total))
    var expected = 0.0
    for y in 0..<(usize)150:
        for x in 0..<(usize)40:
            expected += image.pixels[y * 40 + x] * (f64)(y + 1)
    assert(total == expected)