dymokomi/luce-browser-engine
Source of release 0.6.1
0.6.1 / src / web / loader / in_process_http.lucb
#==============================================================================================
#
# in_process_http - the in-process client's http(s): requests: one HTTP/1.1 exchange on a
# connection, as a state machine on the event loop
#
# DESCRIPTION:
# What curl does for RequestServer with one transfer, without blocking the event loop.
# The request (its method, its headers with Host, Accept-Encoding and the body's
# Content-Length, then its body) is written as the socket takes it; the response is read
# as it arrives, from the connection's Core::Notifier: 1xx heads are skipped, the final
# head is parsed (net.http_parse_response), its headers delivered, and the body, fixed,
# chunked or until the connection closes (net.HttpBodyDecoder), is decoded through its
# Content-Encoding as it comes (in_process_http_encoding.lucb) and delivered chunk by
# chunk, then the end. Everything is delivered on later turns of the event loop
# (in_process_http_delivery.lucb), as RequestServer's replies reach WebContent over IPC.
# Redirects are Fetch's (the 3xx response is delivered). No proxy is used and no cache
# is kept.
#
# Connections are kept alive (HTTP/1.1's default; curl sends no Connection field): after
# a complete response that does not close the connection, the connection returns to the
# pool for the next request to its origin (in_process_http_pool.lucb). A request that
# fails on a reused connection before any byte of its response arrived is retried once
# on a new connection, as curl retries when a kept-alive connection turns out to have
# been closed by the server ("Connection died, retrying a fresh connect"). Errors are
# the network errors RequestServer's curl_code_to_network_error makes of curl's codes.
#
# Not a port: new code of the RequestClient seam (DESIGN.md ยง7.1).
#
# Copyright (c) 2026 the luce-browser authors.
# SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================
# mark: Starting a request ======================================================================
## Whether the request's own headers name `name`.
func in_process_http_has_header(state: const InProcessRequest*, name: str) -> bool:
for header in ak.vector_span_const(&state.request_headers):
let header_name = ak.byte_string_view(&header.name)
if ak.string_view_equals_ignoring_ascii_case(&header_name, ak.sv(name)):
return true
return false
## The request's head: its request line, Host, the request's headers, and the headers the client adds.
func in_process_http_request_head(state: const InProcessRequest*) -> ak.ByteString:
var builder = ak.string_builder_new()
ak.string_builder_append_string_view(&builder, ak.byte_string_view(&state.method))
ak.string_builder_append(&builder, " ")
let path = web_url.url_serialize_path(&state.url)
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&path))
if let query = web_url.url_query(&state.url):
ak.string_builder_append(&builder, "?")
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&query))
ak.string_builder_append(&builder, " HTTP/1.1\r\n")
if not in_process_http_has_header(state, "Host"):
let host = web_url.url_host(&state.url) else trap("VERIFY_NOT_REACHED")
let serialized = web_url.host_serialize(&host)
ak.string_builder_append(&builder, "Host: ")
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&serialized))
if state.origin.port != (u16)(443 if state.origin.secure else 80):
let port_text = ak.string_number_u64((u64)state.origin.port)
ak.string_builder_append(&builder, ":")
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&port_text))
ak.string_builder_append(&builder, "\r\n")
for header in ak.vector_span_const(&state.request_headers):
let name = ak.byte_string_view(&header.name)
if ak.string_view_equals_ignoring_ascii_case(&name, ak.sv("Connection")) or ak.string_view_equals_ignoring_ascii_case(&name, ak.sv("Content-Length")):
continue
ak.string_builder_append_string_view(&builder, name)
ak.string_builder_append(&builder, ": ")
ak.string_builder_append_string_view(&builder, ak.byte_string_view(&header.value))
ak.string_builder_append(&builder, "\r\n")
if not in_process_http_has_header(state, "Accept-Encoding"):
ak.string_builder_append(&builder, "Accept-Encoding: ")
ak.string_builder_append(&builder, in_process_http_accept_encoding)
ak.string_builder_append(&builder, "\r\n")
let method = ak.byte_string_view(&state.method)
if ak.byte_buffer_size(&state.request_body) != 0 or not ak.string_view_eq_string_view(&method, ak.sv("GET")):
let length_text = ak.string_number_u64((u64)ak.byte_buffer_size(&state.request_body))
ak.string_builder_append(&builder, "Content-Length: ")
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&length_text))
ak.string_builder_append(&builder, "\r\n")
ak.string_builder_append(&builder, "\r\n")
return ak.string_builder_to_byte_string(&builder)
## The host to connect to: the URL's host serialized, an IPv6 address without its brackets.
func in_process_http_host(url: const web_url.Url*) -> ak.String?:
let host = web_url.url_host(url) else return none
let serialized = web_url.host_serialize(&host)
let view = ak.string_bytes_as_string_view(&serialized)
let bytes = ak.string_view_bytes(&view)
if bytes.length >= 2 and bytes[0] == '[' and bytes[bytes.length - 1] == ']':
return ak.string_from_utf8(ak.string_view_from_bytes(bytes[1..<bytes.length - 1])) else trap("MUST: String::from_utf8")
return serialized
## Fills the http(s) part of a request's state from its URL: its origin and the bytes it sends; false when the URL
## has no host.
func in_process_http_prepare(state: InProcessRequest*) -> bool:
state.origin = in_process_http_origin_of(&state.url) else return false
let head = in_process_http_request_head(state)
var outgoing = ak.byte_buffer_copy_span_u8(ak.byte_string_bytes(&head)) else trap("out of memory")
ak.byte_buffer_append_span_u8(&outgoing, ak.byte_buffer_bytes_const(&state.request_body))
state.outgoing = outgoing
return true
## An http(s): request (see the header): a connection from the pool, or a new one.
func in_process_http_run(state: InProcessRequest*):
state.start_nanoseconds = in_process_http_now()
if not in_process_http_prepare(state):
in_process_request_finish(state, 0, RequestsRequestTimingInfo(), RequestsNetworkError.malformed_url)
return
in_process_http_pool_acquire(state.client, state, false)
## The client's stop_request for an http(s) request: it leaves the pool's queue, or the connection carrying it closes
## (curl closes a connection whose transfer is abandoned part way).
func in_process_http_stop(state: InProcessRequest*):
if state.waiting:
in_process_http_pool_forget(state.client, state)
return
let connection = state.connection else return
in_process_http_connection_detach(connection)
in_process_http_connection_close(connection)
# mark: The exchange ============================================================================
## The current request's exchange on a connection (the C heap, with the connection).
struct InProcessHttpExchange:
## The request's bytes not yet written.
var outgoing: InProcessHttpBytes
## The response's bytes received and not yet processed.
var incoming: InProcessHttpBytes
## Where the search for the end of the head resumes.
var head_scanned: usize
var head_done: bool
var keep_alive: bool
var until_close: bool
var http_1_0: bool
var body: net.HttpBodyDecoder
var unencoder: InProcessHttpUnencoder
## Bytes of the response received, of its body as sent, and of its body delivered (decoded).
var received: u64
var encoded: u64
var delivered: u64
## When the request was handed this connection, began to be sent, and the response began (monotonic ns).
var acquired_at: u64
var pretransfer_at: u64
var starttransfer_at: u64
## Releases what an exchange holds and makes it ready for the next one.
func in_process_http_exchange_release(exchange: InProcessHttpExchange*):
in_process_http_unencoder_close(&exchange.unencoder)
exchange.outgoing.release()
exchange.incoming.release()
exchange.head_scanned = 0
exchange.head_done = false
exchange.received = 0
exchange.encoded = 0
exchange.delivered = 0
## Starts the exchange of the request `connection` carries: its bytes are written as the socket takes them.
func in_process_http_exchange_begin(connection: InProcessHttpConnection*):
let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
let exchange = &connection.exchange
in_process_http_exchange_release(exchange)
exchange.outgoing.append(ak.byte_buffer_bytes_const(&state.outgoing))
exchange.pretransfer_at = in_process_http_now()
exchange.starttransfer_at = 0
connection.phase = InProcessHttpPhase.sending
in_process_http_connection_arm_timer(connection)
in_process_http_exchange_send(connection)
## Whether a failure of the current exchange should be retried on a new connection: the connection was reused, no
## byte of the response came, and the request was not retried already.
func in_process_http_exchange_retryable(connection: const InProcessHttpConnection*) -> bool:
let state = in_process_http_connection_state(connection) else return false
return connection.reused and connection.exchange.received == 0 and not state.retried
## The request goes to a new connection: this one closes.
func in_process_http_exchange_retry(connection: InProcessHttpConnection*):
let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
let client = connection.client
in_process_http_connection_detach(connection)
state.retried = true
in_process_http_connection_close(connection)
in_process_http_pool_acquire(client, state, true)
## A transfer failed with `code`: retried, or the request finishes with the error (CURLE_SEND_ERROR,
## CURLE_RECV_ERROR: Unknown; a timeout: TimeoutReached).
func in_process_http_exchange_failed(connection: InProcessHttpConnection*, code: ErrorCode):
if in_process_http_exchange_retryable(connection):
in_process_http_exchange_retry(connection)
return
in_process_http_exchange_end(connection, RequestsNetworkError.timeout_reached if code == net.timed_out else RequestsNetworkError.unknown)
## Writes what the socket takes of the request; once all of it is sent, the response is awaited.
func in_process_http_exchange_send(connection: InProcessHttpConnection*):
let exchange = &connection.exchange
while exchange.outgoing.count() > 0:
var blocked = false
var failure: ErrorCode? = none
var sent: usize = 0
with memory.heap:
sent = connection.stream.write(exchange.outgoing.view()) catch problem:
if problem.code == io.would_block:
blocked = true
else:
failure = problem.code
recover 0
if let code = failure:
in_process_http_exchange_failed(connection, code)
return
if blocked:
in_process_http_connection_watch_stream(connection, true)
return
exchange.outgoing.consume(sent)
var flushed = false
var failure: ErrorCode? = none
with memory.heap:
flushed = connection.stream.flush() catch problem:
failure = problem.code
recover false
if let code = failure:
in_process_http_exchange_failed(connection, code)
return
if not flushed:
in_process_http_connection_watch_stream(connection, true)
return
connection.phase = InProcessHttpPhase.receiving
in_process_http_connection_watch_stream(connection, false)
## Reads what has arrived and processes it, until the socket has nothing more or the exchange ends. A burst of reads
## yields to the rest of the event loop; decrypted bytes the stream holds are always read first (no readiness shows
## them).
func in_process_http_exchange_receive(connection: InProcessHttpConnection*):
let exchange = &connection.exchange
var reads: usize = 0
while reads < 16 or connection.stream.pending() > 0:
reads += 1
let room = exchange.incoming.room(16384)
var blocked = false
var failure: ErrorCode? = none
var count: usize = 0
var ended = false
with memory.heap:
count = connection.stream.read(room) catch problem:
if problem.code == io.would_block:
blocked = true
elif problem.code == io.unexpected_eof:
# A TLS server that closed without close_notify: the end of the stream all the same.
ended = true
else:
failure = problem.code
recover 0
if let code = failure:
in_process_http_exchange_failed(connection, code)
return
if blocked:
break
if count == 0 or ended:
in_process_http_exchange_eof(connection)
return
exchange.incoming.grow(count)
if exchange.received == 0:
exchange.starttransfer_at = in_process_http_now()
exchange.received += (u64)count
if in_process_http_exchange_process(connection):
return
in_process_http_connection_watch_stream(connection, false)
## The server closed the connection: the end of an until-close body, or a response cut short.
func in_process_http_exchange_eof(connection: InProcessHttpConnection*):
let exchange = &connection.exchange
if not exchange.head_done:
if in_process_http_exchange_retryable(connection):
in_process_http_exchange_retry(connection)
return
# CURLE_GOT_NOTHING, or a head cut short (CURLE_WEIRD_SERVER_REPLY): Unknown.
in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
return
exchange.keep_alive = false
var truncated = false
exchange.body.finish() catch:
truncated = true
if truncated:
# CURLE_PARTIAL_FILE.
in_process_http_exchange_end(connection, RequestsNetworkError.incomplete_content)
return
let failure = in_process_http_unencoder_finish(&exchange.unencoder)
in_process_http_exchange_end(connection, failure)
## Processes the bytes received: the head, then the body. Answers whether the exchange ended (the connection then
## carries another request or is gone).
func in_process_http_exchange_process(connection: InProcessHttpConnection*) -> bool:
let exchange = &connection.exchange
while true:
if not exchange.head_done:
let available = exchange.incoming.view()
let found = net.http_head_end(available, exchange.head_scanned)
if found == none:
exchange.head_scanned = available.length - 3 if available.length > 3 else 0
if available.length > 65536:
in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
return true
return false
let end = found else trap("VERIFY_NOT_REACHED")
exchange.head_scanned = 0
if in_process_http_exchange_head(connection, available[..<end]):
return true
continue
if exchange.incoming.count() == 0:
return false
var malformed = false
let step = exchange.body.consume(exchange.incoming.view()) catch:
malformed = true
recover net.HttpBodyStep()
if malformed:
# A broken chunk (CURLE_RECV_ERROR): Unknown.
in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
return true
exchange.encoded += (u64)step.body.length
let failure = in_process_http_unencoder_write(&exchange.unencoder, exchange.unencoder.count, step.body)
exchange.incoming.consume(step.consumed)
if failure != none:
in_process_http_exchange_end(connection, failure)
return true
in_process_http_exchange_deliver_output(connection)
if exchange.body.finished():
# Bytes after the response: the connection cannot carry another request.
if exchange.incoming.count() > 0:
exchange.keep_alive = false
in_process_http_exchange_end(connection, in_process_http_unencoder_finish(&exchange.unencoder))
return true
if step.consumed == 0:
return false
## A complete head: skipped when informational; else its headers are delivered and the body prepared. Answers
## whether the exchange ended (a bad head, or no body).
func in_process_http_exchange_head(connection: InProcessHttpConnection*, bytes: const u8[]) -> bool:
let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
let exchange = &connection.exchange
var storage: net.HttpField[100] = ---
let method = ak.string_view_text(ak.byte_string_view(&state.method))
let parsed = in_process_http_parse_head(bytes, storage, method)
if parsed == none:
# CURLE_WEIRD_SERVER_REPLY: Unknown.
in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
return true
let response = parsed else trap("VERIFY_NOT_REACHED")
var invalid = false
if response.status >= 100 and response.status < 200 and response.status != 101:
# An informational head: drop it and wait for the final one.
exchange.incoming.consume(bytes.length)
return false
# The Content-Encoding codings, which a bodiless response (1xx, 204, 304, HEAD) ignores.
let bodiless = response.status < 200 or response.status == 204 or response.status == 304 or net.http_equal(method, "HEAD")
var codings = InProcessHttpCodings()
if not bodiless:
codings = in_process_http_codings(&response)
if codings.too_many:
in_process_http_exchange_end(connection, RequestsNetworkError.invalid_content_encoding)
return true
exchange.body = net.HttpBodyDecoder.create(response.body_kind, response.body_length) catch:
invalid = true
recover net.HttpBodyDecoder()
if invalid:
in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
return true
exchange.unencoder = in_process_http_unencoder_create(&codings)
exchange.keep_alive = response.keep_alive and response.status != 101
exchange.until_close = response.body_kind == net.HttpBodyKind.until_eof
exchange.http_1_0 = response.version == "HTTP/1.0"
exchange.head_done = true
# The headers, copied out of the received bytes, and the reason phrase of the status line.
var headers: ak.Vector[HttpHeader]
for field in response.fields:
ak.vector_append_t(&headers, http_header_create(ak.string_view_from_bytes(field.name.bytes), ak.string_view_from_bytes(field.value.bytes)))
var line_end: usize = 0
while line_end < bytes.length and bytes[line_end] != '\r':
line_end += 1
# "HTTP/1.1 200 OK": the reason phrase starts after the status code.
let reason_start: usize = 13 if line_end > 13 else line_end
let reason_phrase = ak.string_from_utf8(ak.string_view_from_bytes(bytes[reason_start..<line_end])) else ak.String()
in_process_http_post_headers(state, http_header_list_create(headers), (u32)response.status, reason_phrase)
exchange.incoming.consume(bytes.length)
if exchange.body.finished():
in_process_http_exchange_end(connection, none)
return true
return false
## The response head in `bytes`, if it parses.
func in_process_http_parse_head(bytes: const u8[], storage: net.HttpField[], method: str) -> net.HttpHead?:
let response = net.http_parse_response(bytes, storage, method) catch:
return none
return response
## Delivers the body decoded so far.
func in_process_http_exchange_deliver_output(connection: InProcessHttpConnection*):
let exchange = &connection.exchange
let output = exchange.unencoder.output.view()
if output.length == 0:
return
let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
in_process_http_post_data(state, output)
exchange.delivered += (u64)output.length
exchange.unencoder.output.clear()
## The end of the exchange, with `network_error` or none: what was decoded is delivered, then the end; the connection
## returns to the pool if it can carry another request, else it closes.
func in_process_http_exchange_end(connection: InProcessHttpConnection*, network_error: RequestsNetworkError?):
let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
let exchange = &connection.exchange
in_process_http_exchange_deliver_output(connection)
let timing_info = in_process_http_timing_info(connection, state)
let reusable = network_error == none and exchange.head_done and exchange.keep_alive and not exchange.until_close and exchange.incoming.count() == 0 and connection.stream.queued() == 0
in_process_http_connection_detach(connection)
in_process_http_post_finish(state, exchange.delivered, timing_info, network_error)
in_process_http_exchange_release(exchange)
connection.reused = true
if not reusable:
in_process_http_connection_close(connection)
return
in_process_http_connection_unwatch(connection)
connection.phase = InProcessHttpPhase.idle
in_process_http_connection_arm_timer(connection)
in_process_http_pool_release(connection.client, connection)