Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.0

0.6.0 / src / web / loader / in_process_http.lucb

#==============================================================================================
#
#   in_process_http - the in-process client's http(s): requests: one HTTP/1.1 exchange on a
#   connection, as a state machine on the event loop
#
#   DESCRIPTION:
#       What curl does for RequestServer with one transfer, without blocking the event loop.
#       The request (its method, its headers with Host, Accept-Encoding and the body's
#       Content-Length, then its body) is written as the socket takes it; the response is read
#       as it arrives, from the connection's Core::Notifier: 1xx heads are skipped, the final
#       head is parsed (net.http_parse_response), its headers delivered, and the body, fixed,
#       chunked or until the connection closes (net.HttpBodyDecoder), is decoded through its
#       Content-Encoding as it comes (in_process_http_encoding.lucb) and delivered chunk by
#       chunk, then the end. Everything is delivered on later turns of the event loop
#       (in_process_http_delivery.lucb), as RequestServer's replies reach WebContent over IPC.
#       Redirects are Fetch's (the 3xx response is delivered). No proxy is used and no cache
#       is kept.
#
#       Connections are kept alive (HTTP/1.1's default; curl sends no Connection field): after
#       a complete response that does not close the connection, the connection returns to the
#       pool for the next request to its origin (in_process_http_pool.lucb). A request that
#       fails on a reused connection before any byte of its response arrived is retried once
#       on a new connection, as curl retries when a kept-alive connection turns out to have
#       been closed by the server ("Connection died, retrying a fresh connect"). Errors are
#       the network errors RequestServer's curl_code_to_network_error makes of curl's codes.
#
#       Not a port: new code of the RequestClient seam (DESIGN.md ยง7.1).
#
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: Starting a request ======================================================================

## Whether the request's own headers name `name`.
func in_process_http_has_header(state: const InProcessRequest*, name: str) -> bool:
    for header in ak.vector_span_const(&state.request_headers):
        let header_name = ak.byte_string_view(&header.name)
        if ak.string_view_equals_ignoring_ascii_case(&header_name, ak.sv(name)):
            return true
    return false

## The request's head: its request line, Host, the request's headers, and the headers the client adds.
func in_process_http_request_head(state: const InProcessRequest*) -> ak.ByteString:
    var builder = ak.string_builder_new()
    ak.string_builder_append_string_view(&builder, ak.byte_string_view(&state.method))
    ak.string_builder_append(&builder, " ")
    let path = web_url.url_serialize_path(&state.url)
    ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&path))
    if let query = web_url.url_query(&state.url):
        ak.string_builder_append(&builder, "?")
        ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&query))
    ak.string_builder_append(&builder, " HTTP/1.1\r\n")

    if not in_process_http_has_header(state, "Host"):
        let host = web_url.url_host(&state.url) else trap("VERIFY_NOT_REACHED")
        let serialized = web_url.host_serialize(&host)
        ak.string_builder_append(&builder, "Host: ")
        ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&serialized))
        if state.origin.port != (u16)(443 if state.origin.secure else 80):
            let port_text = ak.string_number_u64((u64)state.origin.port)
            ak.string_builder_append(&builder, ":")
            ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&port_text))
        ak.string_builder_append(&builder, "\r\n")
    for header in ak.vector_span_const(&state.request_headers):
        let name = ak.byte_string_view(&header.name)
        if ak.string_view_equals_ignoring_ascii_case(&name, ak.sv("Connection")) or ak.string_view_equals_ignoring_ascii_case(&name, ak.sv("Content-Length")):
            continue
        ak.string_builder_append_string_view(&builder, name)
        ak.string_builder_append(&builder, ": ")
        ak.string_builder_append_string_view(&builder, ak.byte_string_view(&header.value))
        ak.string_builder_append(&builder, "\r\n")
    if not in_process_http_has_header(state, "Accept-Encoding"):
        ak.string_builder_append(&builder, "Accept-Encoding: ")
        ak.string_builder_append(&builder, in_process_http_accept_encoding)
        ak.string_builder_append(&builder, "\r\n")
    let method = ak.byte_string_view(&state.method)
    if ak.byte_buffer_size(&state.request_body) != 0 or not ak.string_view_eq_string_view(&method, ak.sv("GET")):
        let length_text = ak.string_number_u64((u64)ak.byte_buffer_size(&state.request_body))
        ak.string_builder_append(&builder, "Content-Length: ")
        ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&length_text))
        ak.string_builder_append(&builder, "\r\n")
    ak.string_builder_append(&builder, "\r\n")
    return ak.string_builder_to_byte_string(&builder)

## The host to connect to: the URL's host serialized, an IPv6 address without its brackets.
func in_process_http_host(url: const web_url.Url*) -> ak.String?:
    let host = web_url.url_host(url) else return none
    let serialized = web_url.host_serialize(&host)
    let view = ak.string_bytes_as_string_view(&serialized)
    let bytes = ak.string_view_bytes(&view)
    if bytes.length >= 2 and bytes[0] == '[' and bytes[bytes.length - 1] == ']':
        return ak.string_from_utf8(ak.string_view_from_bytes(bytes[1..<bytes.length - 1])) else trap("MUST: String::from_utf8")
    return serialized

## Fills the http(s) part of a request's state from its URL: its origin and the bytes it sends; false when the URL
## has no host.
func in_process_http_prepare(state: InProcessRequest*) -> bool:
    state.origin = in_process_http_origin_of(&state.url) else return false
    let head = in_process_http_request_head(state)
    var outgoing = ak.byte_buffer_copy_span_u8(ak.byte_string_bytes(&head)) else trap("out of memory")
    ak.byte_buffer_append_span_u8(&outgoing, ak.byte_buffer_bytes_const(&state.request_body))
    state.outgoing = outgoing
    return true

## An http(s): request (see the header): a connection from the pool, or a new one.
func in_process_http_run(state: InProcessRequest*):
    state.start_nanoseconds = in_process_http_now()
    if not in_process_http_prepare(state):
        in_process_request_finish(state, 0, RequestsRequestTimingInfo(), RequestsNetworkError.malformed_url)
        return
    in_process_http_pool_acquire(state.client, state, false)

## The client's stop_request for an http(s) request: it leaves the pool's queue, or the connection carrying it closes
## (curl closes a connection whose transfer is abandoned part way).
func in_process_http_stop(state: InProcessRequest*):
    if state.waiting:
        in_process_http_pool_forget(state.client, state)
        return
    let connection = state.connection else return
    in_process_http_connection_detach(connection)
    in_process_http_connection_close(connection)

# mark: The exchange ============================================================================

## The current request's exchange on a connection (the C heap, with the connection).
struct InProcessHttpExchange:
    ## The request's bytes not yet written.
    var outgoing: InProcessHttpBytes
    ## The response's bytes received and not yet processed.
    var incoming: InProcessHttpBytes
    ## Where the search for the end of the head resumes.
    var head_scanned: usize
    var head_done: bool
    var keep_alive: bool
    var until_close: bool
    var http_1_0: bool
    var body: net.HttpBodyDecoder
    var unencoder: InProcessHttpUnencoder
    ## Bytes of the response received, of its body as sent, and of its body delivered (decoded).
    var received: u64
    var encoded: u64
    var delivered: u64
    ## When the request was handed this connection, began to be sent, and the response began (monotonic ns).
    var acquired_at: u64
    var pretransfer_at: u64
    var starttransfer_at: u64

## Releases what an exchange holds and makes it ready for the next one.
func in_process_http_exchange_release(exchange: InProcessHttpExchange*):
    in_process_http_unencoder_close(&exchange.unencoder)
    exchange.outgoing.release()
    exchange.incoming.release()
    exchange.head_scanned = 0
    exchange.head_done = false
    exchange.received = 0
    exchange.encoded = 0
    exchange.delivered = 0

## Starts the exchange of the request `connection` carries: its bytes are written as the socket takes them.
func in_process_http_exchange_begin(connection: InProcessHttpConnection*):
    let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
    let exchange = &connection.exchange
    in_process_http_exchange_release(exchange)
    exchange.outgoing.append(ak.byte_buffer_bytes_const(&state.outgoing))
    exchange.pretransfer_at = in_process_http_now()
    exchange.starttransfer_at = 0
    connection.phase = InProcessHttpPhase.sending
    in_process_http_connection_arm_timer(connection)
    in_process_http_exchange_send(connection)

## Whether a failure of the current exchange should be retried on a new connection: the connection was reused, no
## byte of the response came, and the request was not retried already.
func in_process_http_exchange_retryable(connection: const InProcessHttpConnection*) -> bool:
    let state = in_process_http_connection_state(connection) else return false
    return connection.reused and connection.exchange.received == 0 and not state.retried

## The request goes to a new connection: this one closes.
func in_process_http_exchange_retry(connection: InProcessHttpConnection*):
    let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
    let client = connection.client
    in_process_http_connection_detach(connection)
    state.retried = true
    in_process_http_connection_close(connection)
    in_process_http_pool_acquire(client, state, true)

## A transfer failed with `code`: retried, or the request finishes with the error (CURLE_SEND_ERROR,
## CURLE_RECV_ERROR: Unknown; a timeout: TimeoutReached).
func in_process_http_exchange_failed(connection: InProcessHttpConnection*, code: ErrorCode):
    if in_process_http_exchange_retryable(connection):
        in_process_http_exchange_retry(connection)
        return
    in_process_http_exchange_end(connection, RequestsNetworkError.timeout_reached if code == net.timed_out else RequestsNetworkError.unknown)

## Writes what the socket takes of the request; once all of it is sent, the response is awaited.
func in_process_http_exchange_send(connection: InProcessHttpConnection*):
    let exchange = &connection.exchange
    while exchange.outgoing.count() > 0:
        var blocked = false
        var failure: ErrorCode? = none
        var sent: usize = 0
        with memory.heap:
            sent = connection.stream.write(exchange.outgoing.view()) catch problem:
                if problem.code == io.would_block:
                    blocked = true
                else:
                    failure = problem.code
                recover 0
        if let code = failure:
            in_process_http_exchange_failed(connection, code)
            return
        if blocked:
            in_process_http_connection_watch_stream(connection, true)
            return
        exchange.outgoing.consume(sent)
    var flushed = false
    var failure: ErrorCode? = none
    with memory.heap:
        flushed = connection.stream.flush() catch problem:
            failure = problem.code
            recover false
    if let code = failure:
        in_process_http_exchange_failed(connection, code)
        return
    if not flushed:
        in_process_http_connection_watch_stream(connection, true)
        return
    connection.phase = InProcessHttpPhase.receiving
    in_process_http_connection_watch_stream(connection, false)

## Reads what has arrived and processes it, until the socket has nothing more or the exchange ends. A burst of reads
## yields to the rest of the event loop; decrypted bytes the stream holds are always read first (no readiness shows
## them).
func in_process_http_exchange_receive(connection: InProcessHttpConnection*):
    let exchange = &connection.exchange
    var reads: usize = 0
    while reads < 16 or connection.stream.pending() > 0:
        reads += 1
        let room = exchange.incoming.room(16384)
        var blocked = false
        var failure: ErrorCode? = none
        var count: usize = 0
        var ended = false
        with memory.heap:
            count = connection.stream.read(room) catch problem:
                if problem.code == io.would_block:
                    blocked = true
                elif problem.code == io.unexpected_eof:
                    # A TLS server that closed without close_notify: the end of the stream all the same.
                    ended = true
                else:
                    failure = problem.code
                recover 0
        if let code = failure:
            in_process_http_exchange_failed(connection, code)
            return
        if blocked:
            break
        if count == 0 or ended:
            in_process_http_exchange_eof(connection)
            return
        exchange.incoming.grow(count)
        if exchange.received == 0:
            exchange.starttransfer_at = in_process_http_now()
        exchange.received += (u64)count
        if in_process_http_exchange_process(connection):
            return
    in_process_http_connection_watch_stream(connection, false)

## The server closed the connection: the end of an until-close body, or a response cut short.
func in_process_http_exchange_eof(connection: InProcessHttpConnection*):
    let exchange = &connection.exchange
    if not exchange.head_done:
        if in_process_http_exchange_retryable(connection):
            in_process_http_exchange_retry(connection)
            return
        # CURLE_GOT_NOTHING, or a head cut short (CURLE_WEIRD_SERVER_REPLY): Unknown.
        in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
        return
    exchange.keep_alive = false
    var truncated = false
    exchange.body.finish() catch:
        truncated = true
    if truncated:
        # CURLE_PARTIAL_FILE.
        in_process_http_exchange_end(connection, RequestsNetworkError.incomplete_content)
        return
    let failure = in_process_http_unencoder_finish(&exchange.unencoder)
    in_process_http_exchange_end(connection, failure)

## Processes the bytes received: the head, then the body. Answers whether the exchange ended (the connection then
## carries another request or is gone).
func in_process_http_exchange_process(connection: InProcessHttpConnection*) -> bool:
    let exchange = &connection.exchange
    while true:
        if not exchange.head_done:
            let available = exchange.incoming.view()
            let found = net.http_head_end(available, exchange.head_scanned)
            if found == none:
                exchange.head_scanned = available.length - 3 if available.length > 3 else 0
                if available.length > 65536:
                    in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
                    return true
                return false
            let end = found else trap("VERIFY_NOT_REACHED")
            exchange.head_scanned = 0
            if in_process_http_exchange_head(connection, available[..<end]):
                return true
            continue
        if exchange.incoming.count() == 0:
            return false
        var malformed = false
        let step = exchange.body.consume(exchange.incoming.view()) catch:
            malformed = true
            recover net.HttpBodyStep()
        if malformed:
            # A broken chunk (CURLE_RECV_ERROR): Unknown.
            in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
            return true
        exchange.encoded += (u64)step.body.length
        let failure = in_process_http_unencoder_write(&exchange.unencoder, exchange.unencoder.count, step.body)
        exchange.incoming.consume(step.consumed)
        if failure != none:
            in_process_http_exchange_end(connection, failure)
            return true
        in_process_http_exchange_deliver_output(connection)
        if exchange.body.finished():
            # Bytes after the response: the connection cannot carry another request.
            if exchange.incoming.count() > 0:
                exchange.keep_alive = false
            in_process_http_exchange_end(connection, in_process_http_unencoder_finish(&exchange.unencoder))
            return true
        if step.consumed == 0:
            return false

## A complete head: skipped when informational; else its headers are delivered and the body prepared. Answers
## whether the exchange ended (a bad head, or no body).
func in_process_http_exchange_head(connection: InProcessHttpConnection*, bytes: const u8[]) -> bool:
    let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
    let exchange = &connection.exchange
    var storage: net.HttpField[100] = ---
    let method = ak.string_view_text(ak.byte_string_view(&state.method))
    let parsed = in_process_http_parse_head(bytes, storage, method)
    if parsed == none:
        # CURLE_WEIRD_SERVER_REPLY: Unknown.
        in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
        return true
    let response = parsed else trap("VERIFY_NOT_REACHED")
    var invalid = false
    if response.status >= 100 and response.status < 200 and response.status != 101:
        # An informational head: drop it and wait for the final one.
        exchange.incoming.consume(bytes.length)
        return false

    # The Content-Encoding codings, which a bodiless response (1xx, 204, 304, HEAD) ignores.
    let bodiless = response.status < 200 or response.status == 204 or response.status == 304 or net.http_equal(method, "HEAD")
    var codings = InProcessHttpCodings()
    if not bodiless:
        codings = in_process_http_codings(&response)
        if codings.too_many:
            in_process_http_exchange_end(connection, RequestsNetworkError.invalid_content_encoding)
            return true
    exchange.body = net.HttpBodyDecoder.create(response.body_kind, response.body_length) catch:
        invalid = true
        recover net.HttpBodyDecoder()
    if invalid:
        in_process_http_exchange_end(connection, RequestsNetworkError.unknown)
        return true
    exchange.unencoder = in_process_http_unencoder_create(&codings)
    exchange.keep_alive = response.keep_alive and response.status != 101
    exchange.until_close = response.body_kind == net.HttpBodyKind.until_eof
    exchange.http_1_0 = response.version == "HTTP/1.0"
    exchange.head_done = true

    # The headers, copied out of the received bytes, and the reason phrase of the status line.
    var headers: ak.Vector[HttpHeader]
    for field in response.fields:
        ak.vector_append_t(&headers, http_header_create(ak.string_view_from_bytes(field.name.bytes), ak.string_view_from_bytes(field.value.bytes)))
    var line_end: usize = 0
    while line_end < bytes.length and bytes[line_end] != '\r':
        line_end += 1
    # "HTTP/1.1 200 OK": the reason phrase starts after the status code.
    let reason_start: usize = 13 if line_end > 13 else line_end
    let reason_phrase = ak.string_from_utf8(ak.string_view_from_bytes(bytes[reason_start..<line_end])) else ak.String()
    in_process_http_post_headers(state, http_header_list_create(headers), (u32)response.status, reason_phrase)
    exchange.incoming.consume(bytes.length)
    if exchange.body.finished():
        in_process_http_exchange_end(connection, none)
        return true
    return false

## The response head in `bytes`, if it parses.
func in_process_http_parse_head(bytes: const u8[], storage: net.HttpField[], method: str) -> net.HttpHead?:
    let response = net.http_parse_response(bytes, storage, method) catch:
        return none
    return response

## Delivers the body decoded so far.
func in_process_http_exchange_deliver_output(connection: InProcessHttpConnection*):
    let exchange = &connection.exchange
    let output = exchange.unencoder.output.view()
    if output.length == 0:
        return
    let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
    in_process_http_post_data(state, output)
    exchange.delivered += (u64)output.length
    exchange.unencoder.output.clear()

## The end of the exchange, with `network_error` or none: what was decoded is delivered, then the end; the connection
## returns to the pool if it can carry another request, else it closes.
func in_process_http_exchange_end(connection: InProcessHttpConnection*, network_error: RequestsNetworkError?):
    let state = in_process_http_connection_state(connection) else trap("in-process http: no request")
    let exchange = &connection.exchange
    in_process_http_exchange_deliver_output(connection)
    let timing_info = in_process_http_timing_info(connection, state)
    let reusable = network_error == none and exchange.head_done and exchange.keep_alive and not exchange.until_close and exchange.incoming.count() == 0 and connection.stream.queued() == 0
    in_process_http_connection_detach(connection)
    in_process_http_post_finish(state, exchange.delivered, timing_info, network_error)
    in_process_http_exchange_release(exchange)
    connection.reused = true
    if not reusable:
        in_process_http_connection_close(connection)
        return
    in_process_http_connection_unwatch(connection)
    connection.phase = InProcessHttpPhase.idle
    in_process_http_connection_arm_timer(connection)
    in_process_http_pool_release(connection.client, connection)