Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.1

0.7.1 / src / readahead.lucb

#==============================================================================================
#
#   readahead - Bytes read ahead of the caller
#
#   DESCRIPTION:
#       The buffer behind line, text and chunk reads over a file or a connection: what a read
#       fetched beyond the line or text it answered waits here for the next read.
#
#==============================================================================================

import io
import memory
import utf8

## The text read is not valid UTF-8.
pub let invalid_text: ErrorCode = ErrorCode.package(48)

let smallest: usize = 4096
let kept_capacity: usize = 65536

## Bytes fetched but not yet answered, in a `memory.heap` allocation. The zero value is
## empty; `release` gives the storage back. A view a read answers stays valid until the
## next read through the same buffer.
pub struct ReadAhead:
    var data: u8[]
    var start: usize
    var end: usize

    ## The bytes waiting.
    pub func count() -> usize:
        return self.end - self.start

    ## Move up to `buffer.length` waiting bytes into `buffer`; the number moved.
    pub func take(buffer: u8[]) -> usize:
        let available = self.count()
        let count = buffer.length if buffer.length < available else available
        memory.copy(buffer, self.data[self.start..], count)
        self.start += count
        return count

    ## Forget the waiting bytes, as a seek does.
    pub func clear():
        self.start = 0
        self.end = 0

    ## Give the storage back.
    pub func release():
        if self.data.length > 0:
            free(self.data) in memory.heap
            var empty: u8[]
            self.data = empty
        self.clear()

    ## Room for at least `minimum` more bytes after the waiting ones: the waiting bytes move
    ## to the front, and the storage grows, or shrinks back after a large read.
    func space(minimum: usize) -> u8[]!:
        let waiting = self.count()
        if waiting == 0 and self.data.length > kept_capacity:
            self.release()
        if self.start > 0:
            memory.move(self.data, self.data[self.start..], waiting)
            self.start = 0
            self.end = waiting
        let needed = waiting +? minimum else error(memory.exhausted, "a read is too large")
        if needed > self.data.length:
            var capacity = self.data.length *? 2 else needed
            if capacity < needed:
                capacity = needed
            if capacity < smallest:
                capacity = smallest
            with memory.heap:
                if self.data.length == 0:
                    self.data = try new u8[capacity] ---
                else:
                    self.data = try memory.grow(self.data, capacity)
        return self.data[self.end..]

    ## Read once from `source` into the space after the waiting bytes; false at the end.
    func fill(source: io.Reader, minimum: usize = smallest) -> bool!:
        let room = try self.space(minimum)
        let count = try source.read(room)
        self.end += count
        return count > 0

    ## The next line, its `\n` or `\r\n` consumed and left out; at the end, the unfinished
    ## last line, or none when nothing is left.
    pub func line(source: io.Reader) -> str?!:
        var searched: usize = 0
        while true:
            let waiting = self.data[self.start..<self.end]
            var index = searched
            while index < waiting.length:
                if waiting[index] == '\n':
                    self.start += index + 1
                    let ending = index - 1 if index > 0 and waiting[index - 1] == '\r' else index
                    return try checked(waiting[..<ending])
                index += 1
            searched = waiting.length
            if not try self.fill(source):
                # filling moved the waiting bytes to the front
                let rest = self.data[self.start..<self.end]
                if rest.length == 0:
                    return none
                self.start = self.end
                return try checked(rest)

    ## Up to `maximum` bytes: with `whole`, as many as the source has up to `maximum`,
    ## otherwise what one read brings when nothing is waiting. Empty at the end.
    pub func bytes(source: io.Reader, maximum: usize, whole: bool) -> const u8[]!:
        if self.count() == 0 or whole:
            while self.count() < maximum:
                let wanted = maximum - self.count()
                if not try self.fill(source, wanted if wanted < smallest else smallest):
                    break
                if not whole:
                    break
        let count = maximum if maximum < self.count() else self.count()
        let view = self.data[self.start..<self.start + count]
        self.start += count
        return view

    ## Up to `maximum` bytes as text, as `bytes` reads them, ending on a whole character: a
    ## character the read cut in two waits for the next read, and a maximum too small for
    ## the next character still answers it whole. Invalid UTF-8 is an error.
    pub func text(source: io.Reader, maximum: usize, whole: bool) -> str!:
        if maximum == 0:
            return ""
        var limit = maximum
        while true:
            let view = try self.bytes(source, limit, whole)
            let bad = utf8.invalid_offset(view) else return (str)view
            if not cut_character(view[bad..]):
                error(invalid_text, "the text read is not valid UTF-8")
            self.start -= view.length - bad
            if bad > 0:
                return (str)view[..<bad]
            # nothing but a cut character: reach its end, unless the source has ended
            if limit < 4:
                limit = 4
            elif not try self.fill(source, 4):
                error(invalid_text, "the text read ends inside a character")

## Whether `tail` is the start of a character that more bytes could finish.
func cut_character(tail: const u8[]) -> bool:
    let lead = tail[0]
    let width: usize = 2 if lead >= 194 and lead <= 223 else (3 if lead >= 224 and lead <= 239 else (4 if lead >= 240 and lead <= 244 else 0))
    if width == 0 or tail.length >= width:
        return false
    for byte in tail[1..]:
        if (byte & 192) != 128:
            return false
    return true

func checked(line: const u8[]) -> str!:
    if not utf8.valid(line):
        error(invalid_text, "the line read is not valid UTF-8")
    return (str)line

## A reader over a span, handing out at most `step` bytes a read, for the tests.
struct Chunks: io.Reader:
    var data: const u8[]
    var step: usize

    func read(buffer: u8[]) -> usize!:
        var count = self.step if self.step < self.data.length else self.data.length
        if count > buffer.length:
            count = buffer.length
        memory.copy(buffer, self.data, count)
        self.data = self.data[count..]
        return count

test "lines come whole across reads and the last needs no newline":
    var source = Chunks(data = "one\r\ntwo\n\nthree".bytes, step = 6)
    var ahead: ReadAhead
    defer ahead.release()
    assert(try ahead.line(&source) == "one")
    assert(try ahead.line(&source) == "two")
    assert(try ahead.line(&source) == "")
    assert(try ahead.line(&source) == "three")
    assert(try ahead.line(&source) == none)

test "text ends on a whole character and keeps the rest":
    var source = Chunks(data = "aé€".bytes, step = 2)
    var ahead: ReadAhead
    defer ahead.release()
    assert(try ahead.text(&source, 64, false) == "a")
    assert(try ahead.text(&source, 64, false) == "é")
    assert(try ahead.text(&source, 64, true) == "€")
    assert(try ahead.text(&source, 64, true) == "")

test "bytes stop at the maximum and the rest waits":
    var source = Chunks(data = "abcdefgh".bytes, step = 8)
    var ahead: ReadAhead
    defer ahead.release()
    assert((str)(try ahead.bytes(&source, 3, true)) == "abc")
    var rest: u8[8]
    assert(ahead.take(rest) == 5)

test "invalid text is an error":
    let data: u8[4] = ['a', 255, 'b', '\n']
    var source = Chunks(data = data[..<4], step = 8)
    var ahead: ReadAhead
    defer ahead.release()
    let failed = ahead.line(&source) catch failure:
        assert(failure.code == invalid_text)
        recover none
    assert(failed == none)