Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.0

0.6.0 / src / web / loader / in_process_http_encoding.lucb

#==============================================================================================
#
#   in_process_http_encoding - the in-process client's Content-Encoding: what curl decodes
#   for RequestServer
#
#   DESCRIPTION:
#       Ladybird's RequestServer sets CURLOPT_ACCEPT_ENCODING to "" (Request.cpp), so curl
#       advertises every coding it was built with and decodes the response's Content-Encoding
#       (curl 8.19 lib/content_encoding.c). This fragment does the same with luce-compress:
#
#       - Accept-Encoding: "deflate, gzip, br" when the request names none (curl's list,
#         Curl_get_content_encodings, without zstd, which luce-compress does not decode);
#       - every Content-Encoding field's comma-separated codings stacked in order and
#         decoded last-applied first (Curl_build_unencoding_stack), at most four of them;
#         a bodiless response (1xx, 204, 304, HEAD) ignores them;
#       - identity (and its alias none) passes the body on; deflate is a zlib stream, or
#         raw DEFLATE (with up to four trailing bytes) when the zlib stream fails before
#         giving any output; gzip (and x-gzip) is a gzip member or a zlib stream, as zlib's
#         automatic header detection reads it; br is Brotli;
#       - an unknown coding fails only once body bytes reach it; damaged data fails with
#         InvalidContentEncoding (CURLE_BAD_CONTENT_ENCODING), bytes after a stream's end
#         with Unknown (CURLE_WRITE_ERROR); a stream cut short gives what it decoded.
#
#       The body is decoded as it arrives, as curl's writer stack decodes each piece it is
#       given, through luce-compress's incremental decoders (native's zlib and raw DEFLATE,
#       gzip's member decoder, brotli's decoder): each coding's decoder writes what it makes to
#       the next one, the last to the body delivered. A gzip coding holds its first two bytes
#       until it knows the framing, and a deflate one keeps its input until the zlib stream has
#       given output (to read it again as raw DEFLATE). A decoded body over 1 GiB fails (curl
#       has no bound; this one keeps a small hostile response from exhausting memory). The
#       decoders live with the connection in the C heap (in_process_http_support.lucb).
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: The codings =============================================================================

## A Content-Encoding coding curl knows (its general_unencoders), or one it does not.
enum InProcessHttpCoding:
    identity
    deflate
    gzip
    br
    unknown

## The Accept-Encoding the client sends when the request names none.
let in_process_http_accept_encoding: str = "deflate, gzip, br"

## curl's MAX_ENCODE_STACK: a fifth coding is refused.
let in_process_http_max_codings: usize = 4

## The largest decoded body.
let in_process_http_max_decoded: usize = 1073741824

## The codings of a response, in the order they were applied.
struct InProcessHttpCodings:
    var items: InProcessHttpCoding[4]
    var count: usize
    var too_many: bool

## The coding named `name` (ASCII case-insensitively, as find_unencode_writer matches).
func in_process_http_coding_named(name: str) -> InProcessHttpCoding:
    if net.http_equal(name, "identity") or net.http_equal(name, "none"):
        return InProcessHttpCoding.identity
    if net.http_equal(name, "deflate"):
        return InProcessHttpCoding.deflate
    if net.http_equal(name, "gzip") or net.http_equal(name, "x-gzip"):
        return InProcessHttpCoding.gzip
    if net.http_equal(name, "br"):
        return InProcessHttpCoding.br
    return InProcessHttpCoding.unknown

## The codings of every Content-Encoding field of `response` (Curl_build_unencoding_stack):
## names between commas, leading blanks skipped and trailing whitespace dropped.
func in_process_http_codings(response: const net.HttpHead*) -> InProcessHttpCodings:
    var codings = InProcessHttpCodings()
    for field in response.fields:
        if not net.http_equal(field.name, "content-encoding"):
            continue
        let text = field.value.bytes
        var at: usize = 0
        while at < text.length:
            while at < text.length and (text[at] == ' ' or text[at] == '\t' or text[at] == ','):
                at += 1
            let start = at
            var end = at
            while at < text.length and text[at] != ',':
                if text[at] > ' ':
                    end = at + 1
                at += 1
            if end == start:
                continue
            if codings.count == in_process_http_max_codings:
                codings.too_many = true
                return codings
            codings.items[codings.count] = in_process_http_coding_named((str)text[start..<end])
            codings.count += 1
    return codings

# mark: Decoding ================================================================================

## Which luce-compress decoder an InProcessHttpDecoder runs.
enum InProcessHttpDecoderKind:
    zlib
    raw
    gzip
    brotli

## One coding's incremental decoder.
struct InProcessHttpDecoder:
    var kind: InProcessHttpDecoderKind
    var inflater: compress_native.Decoder
    var member: compress_gzip.Decoder
    var brotli: compress_brotli.Decoder

    ## A decoder of `kind`, in the C heap.
    static func open(kind: InProcessHttpDecoderKind) -> InProcessHttpDecoder!:
        with memory.heap:
            var made = InProcessHttpDecoder(kind = kind)
            if kind == InProcessHttpDecoderKind.zlib or kind == InProcessHttpDecoderKind.raw:
                let framing = "zlib" if kind == InProcessHttpDecoderKind.zlib else "raw"
                made.inflater = try compress_native.make_decoder(framing, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF)
            elif kind == InProcessHttpDecoderKind.gzip:
                made.member = compress_gzip.Decoder.open(multiple = false)
            else:
                made.brotli = compress_brotli.Decoder.open()
            return made

    ## One step: some of `input` into `output`.
    func step(input: const u8[], output: u8[]) -> compress_native.Step!:
        with memory.heap:
            if self.kind == InProcessHttpDecoderKind.gzip:
                return try self.member.step(input, output)
            if self.kind == InProcessHttpDecoderKind.brotli:
                return try self.brotli.step(input, output)
            return try self.inflater.step(input, output)

    ## Release what the decoder holds (Brotli's window and prefix codes).
    func close():
        with memory.heap:
            if self.kind == InProcessHttpDecoderKind.gzip:
                self.member.close()
            elif self.kind == InProcessHttpDecoderKind.brotli:
                self.brotli.close()
            else:
                self.inflater.close()

## A decoding failure as the network error curl's writer answers: luce-compress's failures
## (damaged data) are a bad content encoding, a body too large is Unknown
## (CURLE_OUT_OF_MEMORY).
func in_process_http_decode_error(code: ErrorCode) -> RequestsNetworkError:
    if code == ak.ak_error:
        return RequestsNetworkError.unknown
    return RequestsNetworkError.invalid_content_encoding

# mark: The writer stack ========================================================================

## One coding's writer (curl's unencoder): its decoder once the framing is known, the input it
## holds back, and whether its stream has ended.
struct InProcessHttpStage:
    var coding: InProcessHttpCoding
    var opened: bool
    var decoder: InProcessHttpDecoder
    ## Input kept: gzip's first bytes until the framing is known; deflate's input until the zlib
    ## stream has given output, to be read again as raw DEFLATE if it fails before that.
    var held: InProcessHttpBytes
    var produced: bool
    var raw_retry: bool
    var finished: bool
    ## Bytes after the end of the stream, and how many are tolerated (4 after a raw retry).
    var trailing: usize

## The codings of a response as a stack of writers, the last applied first, and the decoded body
## not yet delivered (the C heap; closed with in_process_http_unencoder_close).
struct InProcessHttpUnencoder:
    var stages: InProcessHttpStage[4]
    var count: usize
    var output: InProcessHttpBytes
    var decoded: usize

## The writer stack of `codings` (none of them when the response has no coding).
func in_process_http_unencoder_create(codings: const InProcessHttpCodings*) -> InProcessHttpUnencoder:
    var unencoder = InProcessHttpUnencoder(count = codings.count)
    for index in 0..<codings.count:
        unencoder.stages[index].coding = codings.items[index]
    return unencoder

## Releases what the stack holds: its decoders and buffers.
func in_process_http_unencoder_close(this: InProcessHttpUnencoder*):
    for index in 0..<this.count:
        let stage = &this.stages[index]
        if stage.opened:
            stage.decoder.close()
            stage.opened = false
        stage.held.release()
    this.count = 0
    this.output.release()

## `data` through the writers below stage `depth` (depth 0 is the decoded body): what the last
## applied coding decodes goes to the one applied before it. Answers the network error, if any.
func in_process_http_unencoder_write(this: InProcessHttpUnencoder*, depth: usize, data: const u8[]) -> RequestsNetworkError?:
    if data.length == 0:
        return none
    if depth == 0:
        if data.length > in_process_http_max_decoded - this.decoded:
            return RequestsNetworkError.unknown
        this.decoded += data.length
        this.output.append(data)
        return none
    let stage = &this.stages[depth - 1]
    if stage.coding == InProcessHttpCoding.identity:
        return in_process_http_unencoder_write(this, depth - 1, data)
    if stage.coding == InProcessHttpCoding.unknown:
        # The deferred error writer: "Unrecognized content encoding type".
        return RequestsNetworkError.invalid_content_encoding
    if stage.finished:
        # Bytes after the end of the stream: process_trailer's (and brotli's) write error.
        stage.trailing += data.length
        if stage.trailing > (4 if stage.raw_retry else 0):
            return RequestsNetworkError.unknown
        return none
    if not stage.opened:
        if stage.coding == InProcessHttpCoding.gzip and stage.held.count() + data.length < 2:
            stage.held.append(data)
            return none
        # inflateInit2(MAX_WBITS + 32): a gzip header, or else a zlib one.
        var kind = InProcessHttpDecoderKind.zlib
        if stage.coding == InProcessHttpCoding.br:
            kind = InProcessHttpDecoderKind.brotli
        elif stage.coding == InProcessHttpCoding.gzip:
            stage.held.append(data)
            let first = stage.held.view()
            kind = InProcessHttpDecoderKind.gzip if first[0] == 0x1F and first[1] == 0x8B else InProcessHttpDecoderKind.zlib
        if let failure = in_process_http_stage_open(stage, kind):
            return failure
        if stage.coding == InProcessHttpCoding.gzip:
            # The held bytes (now with `data`) go through the decoder at once.
            var held = stage.held
            stage.held = InProcessHttpBytes()
            defer held.release()
            return in_process_http_unencoder_feed(this, depth, held.view())
    if stage.coding == InProcessHttpCoding.deflate and not stage.produced and not stage.raw_retry:
        stage.held.append(data)
    return in_process_http_unencoder_feed(this, depth, data)

## Opens the decoder of `stage`.
func in_process_http_stage_open(stage: InProcessHttpStage*, kind: InProcessHttpDecoderKind) -> RequestsNetworkError?:
    stage.decoder = InProcessHttpDecoder.open(kind) catch failure:
        return in_process_http_decode_error(failure.code)
    stage.opened = true
    return none

## `data` through the open decoder of stage `depth`, its output to the writers below.
func in_process_http_unencoder_feed(this: InProcessHttpUnencoder*, depth: usize, data: const u8[]) -> RequestsNetworkError?:
    let stage = &this.stages[depth - 1]
    var chunk: u8[16384] = ---
    var at: usize = 0
    while true:
        let step = stage.decoder.step(data[at..], chunk) catch failure:
            # "some servers seem to not generate zlib headers": a deflate body whose zlib
            # stream fails before any output is read again as raw DEFLATE, tolerating 4
            # trailing bytes.
            if stage.coding != InProcessHttpCoding.deflate or stage.produced or stage.raw_retry:
                return in_process_http_decode_error(failure.code)
            stage.decoder.close()
            stage.opened = false
            stage.raw_retry = true
            if let reopened = in_process_http_stage_open(stage, InProcessHttpDecoderKind.raw):
                return reopened
            var held = stage.held
            stage.held = InProcessHttpBytes()
            defer held.release()
            return in_process_http_unencoder_feed(this, depth, held.view())
        at += (usize)step.consumed
        let produced = (usize)step.produced
        if produced > 0:
            if not stage.produced:
                stage.produced = true
                stage.held.release()
            if let failure = in_process_http_unencoder_write(this, depth - 1, chunk[..<produced]):
                return failure
        if step.status == compress_native.finished:
            stage.finished = true
            return in_process_http_unencoder_write(this, depth, data[at..])
        if step.status == compress_native.need_input:
            return none
        if step.status == compress_native.need_output and step.consumed == 0 and produced == 0:
            return in_process_http_decode_error(ak.ak_error)

## The end of the body: a gzip coding still holding its first byte decodes it (as zlib), from the
## last applied coding to the first. A stream cut short gives what it decoded, without an error.
func in_process_http_unencoder_finish(this: InProcessHttpUnencoder*) -> RequestsNetworkError?:
    var depth = this.count
    while depth > 0:
        let stage = &this.stages[depth - 1]
        if not stage.opened and stage.held.count() > 0:
            if let failure = in_process_http_stage_open(stage, InProcessHttpDecoderKind.zlib):
                return failure
            var held = stage.held
            stage.held = InProcessHttpBytes()
            defer held.release()
            if let failure = in_process_http_unencoder_feed(this, depth, held.view()):
                return failure
        depth -= 1
    return none