dymokomi/luce-browser-engine
Source of release 0.6.0
0.6.0 / src / web / loader / in_process_http_encoding.lucb
#==============================================================================================
#
# in_process_http_encoding - the in-process client's Content-Encoding: what curl decodes
# for RequestServer
#
# DESCRIPTION:
# Ladybird's RequestServer sets CURLOPT_ACCEPT_ENCODING to "" (Request.cpp), so curl
# advertises every coding it was built with and decodes the response's Content-Encoding
# (curl 8.19 lib/content_encoding.c). This fragment does the same with luce-compress:
#
# - Accept-Encoding: "deflate, gzip, br" when the request names none (curl's list,
# Curl_get_content_encodings, without zstd, which luce-compress does not decode);
# - every Content-Encoding field's comma-separated codings stacked in order and
# decoded last-applied first (Curl_build_unencoding_stack), at most four of them;
# a bodiless response (1xx, 204, 304, HEAD) ignores them;
# - identity (and its alias none) passes the body on; deflate is a zlib stream, or
# raw DEFLATE (with up to four trailing bytes) when the zlib stream fails before
# giving any output; gzip (and x-gzip) is a gzip member or a zlib stream, as zlib's
# automatic header detection reads it; br is Brotli;
# - an unknown coding fails only once body bytes reach it; damaged data fails with
# InvalidContentEncoding (CURLE_BAD_CONTENT_ENCODING), bytes after a stream's end
# with Unknown (CURLE_WRITE_ERROR); a stream cut short gives what it decoded.
#
# The body is decoded as it arrives, as curl's writer stack decodes each piece it is
# given, through luce-compress's incremental decoders (native's zlib and raw DEFLATE,
# gzip's member decoder, brotli's decoder): each coding's decoder writes what it makes to
# the next one, the last to the body delivered. A gzip coding holds its first two bytes
# until it knows the framing, and a deflate one keeps its input until the zlib stream has
# given output (to read it again as raw DEFLATE). A decoded body over 1 GiB fails (curl
# has no bound; this one keeps a small hostile response from exhausting memory). The
# decoders live with the connection in the C heap (in_process_http_support.lucb).
#
# Copyright (c) 2026 the luce-browser authors.
# SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================
# mark: The codings =============================================================================
## A Content-Encoding coding curl knows (its general_unencoders), or one it does not.
enum InProcessHttpCoding:
identity
deflate
gzip
br
unknown
## The Accept-Encoding the client sends when the request names none.
let in_process_http_accept_encoding: str = "deflate, gzip, br"
## curl's MAX_ENCODE_STACK: a fifth coding is refused.
let in_process_http_max_codings: usize = 4
## The largest decoded body.
let in_process_http_max_decoded: usize = 1073741824
## The codings of a response, in the order they were applied.
struct InProcessHttpCodings:
var items: InProcessHttpCoding[4]
var count: usize
var too_many: bool
## The coding named `name` (ASCII case-insensitively, as find_unencode_writer matches).
func in_process_http_coding_named(name: str) -> InProcessHttpCoding:
if net.http_equal(name, "identity") or net.http_equal(name, "none"):
return InProcessHttpCoding.identity
if net.http_equal(name, "deflate"):
return InProcessHttpCoding.deflate
if net.http_equal(name, "gzip") or net.http_equal(name, "x-gzip"):
return InProcessHttpCoding.gzip
if net.http_equal(name, "br"):
return InProcessHttpCoding.br
return InProcessHttpCoding.unknown
## The codings of every Content-Encoding field of `response` (Curl_build_unencoding_stack):
## names between commas, leading blanks skipped and trailing whitespace dropped.
func in_process_http_codings(response: const net.HttpHead*) -> InProcessHttpCodings:
var codings = InProcessHttpCodings()
for field in response.fields:
if not net.http_equal(field.name, "content-encoding"):
continue
let text = field.value.bytes
var at: usize = 0
while at < text.length:
while at < text.length and (text[at] == ' ' or text[at] == '\t' or text[at] == ','):
at += 1
let start = at
var end = at
while at < text.length and text[at] != ',':
if text[at] > ' ':
end = at + 1
at += 1
if end == start:
continue
if codings.count == in_process_http_max_codings:
codings.too_many = true
return codings
codings.items[codings.count] = in_process_http_coding_named((str)text[start..<end])
codings.count += 1
return codings
# mark: Decoding ================================================================================
## Which luce-compress decoder an InProcessHttpDecoder runs.
enum InProcessHttpDecoderKind:
zlib
raw
gzip
brotli
## One coding's incremental decoder.
struct InProcessHttpDecoder:
var kind: InProcessHttpDecoderKind
var inflater: compress_native.Decoder
var member: compress_gzip.Decoder
var brotli: compress_brotli.Decoder
## A decoder of `kind`, in the C heap.
static func open(kind: InProcessHttpDecoderKind) -> InProcessHttpDecoder!:
with memory.heap:
var made = InProcessHttpDecoder(kind = kind)
if kind == InProcessHttpDecoderKind.zlib or kind == InProcessHttpDecoderKind.raw:
let framing = "zlib" if kind == InProcessHttpDecoderKind.zlib else "raw"
made.inflater = try compress_native.make_decoder(framing, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF)
elif kind == InProcessHttpDecoderKind.gzip:
made.member = compress_gzip.Decoder.open(multiple = false)
else:
made.brotli = compress_brotli.Decoder.open()
return made
## One step: some of `input` into `output`.
func step(input: const u8[], output: u8[]) -> compress_native.Step!:
with memory.heap:
if self.kind == InProcessHttpDecoderKind.gzip:
return try self.member.step(input, output)
if self.kind == InProcessHttpDecoderKind.brotli:
return try self.brotli.step(input, output)
return try self.inflater.step(input, output)
## Release what the decoder holds (Brotli's window and prefix codes).
func close():
with memory.heap:
if self.kind == InProcessHttpDecoderKind.gzip:
self.member.close()
elif self.kind == InProcessHttpDecoderKind.brotli:
self.brotli.close()
else:
self.inflater.close()
## A decoding failure as the network error curl's writer answers: luce-compress's failures
## (damaged data) are a bad content encoding, a body too large is Unknown
## (CURLE_OUT_OF_MEMORY).
func in_process_http_decode_error(code: ErrorCode) -> RequestsNetworkError:
if code == ak.ak_error:
return RequestsNetworkError.unknown
return RequestsNetworkError.invalid_content_encoding
# mark: The writer stack ========================================================================
## One coding's writer (curl's unencoder): its decoder once the framing is known, the input it
## holds back, and whether its stream has ended.
struct InProcessHttpStage:
var coding: InProcessHttpCoding
var opened: bool
var decoder: InProcessHttpDecoder
## Input kept: gzip's first bytes until the framing is known; deflate's input until the zlib
## stream has given output, to be read again as raw DEFLATE if it fails before that.
var held: InProcessHttpBytes
var produced: bool
var raw_retry: bool
var finished: bool
## Bytes after the end of the stream, and how many are tolerated (4 after a raw retry).
var trailing: usize
## The codings of a response as a stack of writers, the last applied first, and the decoded body
## not yet delivered (the C heap; closed with in_process_http_unencoder_close).
struct InProcessHttpUnencoder:
var stages: InProcessHttpStage[4]
var count: usize
var output: InProcessHttpBytes
var decoded: usize
## The writer stack of `codings` (none of them when the response has no coding).
func in_process_http_unencoder_create(codings: const InProcessHttpCodings*) -> InProcessHttpUnencoder:
var unencoder = InProcessHttpUnencoder(count = codings.count)
for index in 0..<codings.count:
unencoder.stages[index].coding = codings.items[index]
return unencoder
## Releases what the stack holds: its decoders and buffers.
func in_process_http_unencoder_close(this: InProcessHttpUnencoder*):
for index in 0..<this.count:
let stage = &this.stages[index]
if stage.opened:
stage.decoder.close()
stage.opened = false
stage.held.release()
this.count = 0
this.output.release()
## `data` through the writers below stage `depth` (depth 0 is the decoded body): what the last
## applied coding decodes goes to the one applied before it. Answers the network error, if any.
func in_process_http_unencoder_write(this: InProcessHttpUnencoder*, depth: usize, data: const u8[]) -> RequestsNetworkError?:
if data.length == 0:
return none
if depth == 0:
if data.length > in_process_http_max_decoded - this.decoded:
return RequestsNetworkError.unknown
this.decoded += data.length
this.output.append(data)
return none
let stage = &this.stages[depth - 1]
if stage.coding == InProcessHttpCoding.identity:
return in_process_http_unencoder_write(this, depth - 1, data)
if stage.coding == InProcessHttpCoding.unknown:
# The deferred error writer: "Unrecognized content encoding type".
return RequestsNetworkError.invalid_content_encoding
if stage.finished:
# Bytes after the end of the stream: process_trailer's (and brotli's) write error.
stage.trailing += data.length
if stage.trailing > (4 if stage.raw_retry else 0):
return RequestsNetworkError.unknown
return none
if not stage.opened:
if stage.coding == InProcessHttpCoding.gzip and stage.held.count() + data.length < 2:
stage.held.append(data)
return none
# inflateInit2(MAX_WBITS + 32): a gzip header, or else a zlib one.
var kind = InProcessHttpDecoderKind.zlib
if stage.coding == InProcessHttpCoding.br:
kind = InProcessHttpDecoderKind.brotli
elif stage.coding == InProcessHttpCoding.gzip:
stage.held.append(data)
let first = stage.held.view()
kind = InProcessHttpDecoderKind.gzip if first[0] == 0x1F and first[1] == 0x8B else InProcessHttpDecoderKind.zlib
if let failure = in_process_http_stage_open(stage, kind):
return failure
if stage.coding == InProcessHttpCoding.gzip:
# The held bytes (now with `data`) go through the decoder at once.
var held = stage.held
stage.held = InProcessHttpBytes()
defer held.release()
return in_process_http_unencoder_feed(this, depth, held.view())
if stage.coding == InProcessHttpCoding.deflate and not stage.produced and not stage.raw_retry:
stage.held.append(data)
return in_process_http_unencoder_feed(this, depth, data)
## Opens the decoder of `stage`.
func in_process_http_stage_open(stage: InProcessHttpStage*, kind: InProcessHttpDecoderKind) -> RequestsNetworkError?:
stage.decoder = InProcessHttpDecoder.open(kind) catch failure:
return in_process_http_decode_error(failure.code)
stage.opened = true
return none
## `data` through the open decoder of stage `depth`, its output to the writers below.
func in_process_http_unencoder_feed(this: InProcessHttpUnencoder*, depth: usize, data: const u8[]) -> RequestsNetworkError?:
let stage = &this.stages[depth - 1]
var chunk: u8[16384] = ---
var at: usize = 0
while true:
let step = stage.decoder.step(data[at..], chunk) catch failure:
# "some servers seem to not generate zlib headers": a deflate body whose zlib
# stream fails before any output is read again as raw DEFLATE, tolerating 4
# trailing bytes.
if stage.coding != InProcessHttpCoding.deflate or stage.produced or stage.raw_retry:
return in_process_http_decode_error(failure.code)
stage.decoder.close()
stage.opened = false
stage.raw_retry = true
if let reopened = in_process_http_stage_open(stage, InProcessHttpDecoderKind.raw):
return reopened
var held = stage.held
stage.held = InProcessHttpBytes()
defer held.release()
return in_process_http_unencoder_feed(this, depth, held.view())
at += (usize)step.consumed
let produced = (usize)step.produced
if produced > 0:
if not stage.produced:
stage.produced = true
stage.held.release()
if let failure = in_process_http_unencoder_write(this, depth - 1, chunk[..<produced]):
return failure
if step.status == compress_native.finished:
stage.finished = true
return in_process_http_unencoder_write(this, depth, data[at..])
if step.status == compress_native.need_input:
return none
if step.status == compress_native.need_output and step.consumed == 0 and produced == 0:
return in_process_http_decode_error(ak.ak_error)
## The end of the body: a gzip coding still holding its first byte decodes it (as zlib), from the
## last applied coding to the first. A stream cut short gives what it decoded, without an error.
func in_process_http_unencoder_finish(this: InProcessHttpUnencoder*) -> RequestsNetworkError?:
var depth = this.count
while depth > 0:
let stage = &this.stages[depth - 1]
if not stage.opened and stage.held.count() > 0:
if let failure = in_process_http_stage_open(stage, InProcessHttpDecoderKind.zlib):
return failure
var held = stage.held
stage.held = InProcessHttpBytes()
defer held.release()
if let failure = in_process_http_unencoder_feed(this, depth, held.view()):
return failure
depth -= 1
return none