Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.1

0.6.1 / src / web / fetch / fetching / fetching_1.lucb

#==============================================================================================
#
#   fetching_1 - Fetch::Fetching: the HTTP cache partitions, fetch, and populating a request
#   from its client
#
#   DESCRIPTION:
#       Port of Libraries/LibWeb/Fetch/Fetching/Fetching.cpp (lines 75-381): HTTPCache (one
#       HTTP::MemoryCache per network partition key, for the process), determining the HTTP
#       cache partition, selecting a response from and storing one in the memory cache (off
#       unless enabled), the fetch algorithm (Fetching.h's entry point) and populate request
#       from client. Main fetch is in fetching_2, fetch response handover in fetching_3,
#       scheme fetch in fetching_4, HTTP fetch and HTTP-redirect fetch in fetching_5,
#       HTTP-network-or-cache fetch in fetching_6, the network fetch through the
#       ResourceLoader, CORS-preflight fetch and the Fetch metadata headers in fetching_7.
#
#       Copyright (c) 2022-2023, Linus Groh <linusg@serenityos.org>
#       Copyright (c) 2023, Luke Wilde <lukew@serenityos.org>
#       Copyright (c) 2023, Sam Atkins <atkinssj@serenityos.org>
#       Copyright (c) 2024, Jamie Mansfield <jmansfield@cadixdev.org>
#       Copyright (c) 2025, Shannon Booth <shannon@serenityos.org>
#       Copyright (c) 2025, Kenneth Myhra <kennethmyhra@serenityos.org>
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: The HTTP cache ==========================================================================

## HTTPCache::the()'s `static HTTPCache s_cache`: for the process, so its map and what it keeps live in the C heap
## (DESIGN.md §3.3 rule 7).
var fetch_http_cache_s_cache: FetchHttpCache

## A copy of `value` in the C heap, for a key of the HTTP cache. Runs with the C heap current and ak's atomic
## allocator unset (fetch_http_cache_get).
func fetch_http_cache_kept_string(value: ak.String) -> ak.String:
    return ak.string_from_utf8(ak.string_bytes_as_string_view(&value)) else trap("MUST: String::from_utf8")

## A copy of `value` in the C heap, as fetch_http_cache_kept_string.
func fetch_http_cache_kept_optional_string(value: ak.String?) -> ak.String?:
    if let string = value:
        return fetch_http_cache_kept_string(string)
    return none

## A copy of a network partition key whose strings live in the C heap, for HTTPCache's map: the key's origin
## (an opaque origin holds no strings).
func fetch_http_cache_kept_key(key: const FetchNetworkPartitionKey*) -> FetchNetworkPartitionKey:
    match key.top_level_origin.m_state:
        .origin_opaque_data(_):
            return *key
        .origin_tuple(tuple):
            let host_value: ak.HostVariantType = match tuple.host.m_value:
                .string(string) => .string(fetch_http_cache_kept_string(string))
                _ => tuple.host.m_value
            let origin = web_url.origin_tuple(fetch_http_cache_kept_optional_string(tuple.scheme), web_url.Host(m_value = host_value), tuple.port, fetch_http_cache_kept_optional_string(tuple.domain))
            return FetchNetworkPartitionKey(top_level_origin = origin, second_key = key.second_key)

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:89 Web::Fetch::Fetching::HTTPCache::get.
pub func fetch_http_cache_get(this: FetchHttpCache*, key: const FetchNetworkPartitionKey*) -> HttpMemoryCache*:
    # return *m_cache.ensure(key, [] { return HTTP::MemoryCache::create(); }); the entry and its key in the C heap.
    if let cache = ak.hash_map_get_k(&this.m_cache, *key):
        return *cache
    let saved_atomic_allocator = ak.atomic_allocator
    ak.atomic_allocator = none
    defer ak.atomic_allocator = saved_atomic_allocator
    with memory.heap:
        let kept_key = fetch_http_cache_kept_key(key)
        let cache = http_memory_cache_create()
        ak.hash_map_set(&this.m_cache, kept_key, cache)
        return cache

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:96 Web::Fetch::Fetching::HTTPCache::the.
pub func fetch_http_cache_the() -> FetchHttpCache*:
    return &fetch_http_cache_s_cache

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:102 Web::Fetch::Fetching::HTTPCache::clear_cache.
pub func fetch_http_cache_clear_cache(this: FetchHttpCache*):
    # (The map's storage is in the C heap; the keys' strings and the caches are never freed, as the C++
    #  process-wide objects.)
    let saved_atomic_allocator = ak.atomic_allocator
    ak.atomic_allocator = none
    defer ak.atomic_allocator = saved_atomic_allocator
    with memory.heap:
        ak.hash_map_clear(&this.m_cache)

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:112 Web::Fetch::Fetching::determine_the_http_cache_partition.
# https://fetch.spec.whatwg.org/#determine-the-http-cache-partition
func fetch_fetching_determine_the_http_cache_partition(request: FetchRequest*) -> HttpMemoryCache*?:
    # 1. Let key be the result of determining the network partition key given request.
    # 2. If key is null, then return null.
    let key = fetch_determine_the_network_partition_key_fetch_request(request) else return none

    # 3. Return the unique HTTP cache associated with key. [HTTP-CACHING]
    return fetch_http_cache_get(fetch_http_cache_the(), &key)

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:125 Web::Fetch::Fetching::select_response_from_cache.
func fetch_fetching_select_response_from_cache(realm: Realm*, http_cache: HttpMemoryCache*, request: FetchRequest*) -> FetchResponse*?:
    if not fetch_fetching_g_http_memory_cache_enabled:
        return none

    let method = fetch_request_method(request)
    let cache_entry = http_memory_cache_open_entry(http_cache, fetch_request_current_url(request), ak.byte_string_view(&method), fetch_request_header_list(request), fetch_request_cache_mode(request)) else return none

    let response = fetch_infrastructure_response_create(realm_vm(realm))
    ak.vector_append_t(fetch_response_url_list(response), fetch_request_current_url(request))
    fetch_response_set_method(response, method)

    fetch_response_set_status(response, (u16)cache_entry.status_code)
    fetch_response_set_status_message(response, cache_entry.reason_phrase)
    fetch_response_set_header_list(response, cache_entry.response_headers)

    let body_with_type = fetch_safely_extract_body_bytes(realm, ak.byte_buffer_bytes_const(&cache_entry.response_body))
    fetch_response_set_body(response, body_with_type.body)

    return response

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:148 Web::Fetch::Fetching::store_response_in_cache.
func fetch_fetching_store_response_in_cache(http_cache: HttpMemoryCache*, request: FetchRequest*, response: FetchResponse*):
    if not fetch_fetching_g_http_memory_cache_enabled:
        return
    if fetch_request_cache_mode(request) == HttpCacheMode.no_store:
        return

    let method = fetch_request_method(request)
    http_memory_cache_create_entry(http_cache, fetch_request_current_url(request), ak.byte_string_view(&method), fetch_request_header_list(request), fetch_request_request_time(request), (u32)fetch_response_status(response), fetch_response_status_message(response), fetch_response_header_list(response))

# mark: Fetch ===================================================================================

## The captures of fetch's onPreloadedResponseAvailable, `[fetch_params](GC::Ref<Response> response)`.
struct FetchFetchingFetchOnPreloadedResponseAvailable:
    var fetch_params: FetchParams*

    ## Sets fetchParams’s preloaded response candidate to response.
    static func call(context: void*?, response: FetchResponse*):
        let captures = (FetchFetchingFetchOnPreloadedResponseAvailable*)(context else trap("fetch: no captures"))
        fetch_params_set_preloaded_response_candidate(captures.fetch_params, .fetch_response(response))

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:159 Web::Fetch::Fetching::fetch.
# https://fetch.spec.whatwg.org/#concept-fetch
pub func fetch_fetching_fetch(realm: Realm*, request: FetchRequest*, algorithms: const FetchAlgorithms*, use_parallel_queue: FetchUseParallelQueue = FetchUseParallelQueue.no) -> FetchController*:
    if fetch_web_fetch_debug:
        ak.dbgln(f"Fetch: Running 'fetch' with: request @ {(void*)request}")

    let vm = realm_vm(realm)

    # 1. Assert: request’s mode is "navigate" or processEarlyHintsResponse is null.
    let process_early_hints_response = fetch_algorithms_process_early_hints_response((FetchAlgorithms*)algorithms)
    assert(fetch_request_mode(request) == FetchRequestMode.navigate or not ak.function1_is_set(&process_early_hints_response))

    # 2. Let taskDestination be null.
    var task_destination: FetchTaskDestination = .empty

    # 3. Let crossOriginIsolatedCapability be false.
    var cross_origin_isolated_capability = HtmlCanUseCrossOriginIsolatedApIs.no

    # 4. Populate request from client given request.
    fetch_fetching_populate_request_from_client(realm, request)

    # 5. If request’s client is non-null, then:
    if let client = fetch_request_client(request):
        # 1. Set taskDestination to request’s client’s global object.
        task_destination = .js_object(html_environment_settings_object_global_object(client))

        # 2. Set crossOriginIsolatedCapability to request’s client’s cross-origin isolated capability.
        cross_origin_isolated_capability = html_environment_settings_object_cross_origin_isolated_capability(client)

    # 6. If useParallelQueue is true, then set taskDestination to the result of starting a new parallel queue.
    if use_parallel_queue == FetchUseParallelQueue.yes:
        task_destination = .html_parallel_queue(html_parallel_queue_create())

    # 7. Let timingInfo be a new fetch timing info whose start time and post-redirect start time are the coarsened
    #    shared current time given crossOriginIsolatedCapability, and render-blocking is set to request’s
    #    render-blocking.
    let timing_info = fetch_timing_info_create(vm)
    let now = high_resolution_time_coarsened_shared_current_time(cross_origin_isolated_capability)
    fetch_timing_info_set_start_time(timing_info, now)
    fetch_timing_info_set_post_redirect_start_time(timing_info, now)
    fetch_timing_info_set_render_blocking(timing_info, fetch_request_render_blocking(request))

    # 8. Let fetchParams be a new fetch params whose request is request, timing info is timingInfo, process request
    #    body chunk length is processRequestBodyChunkLength, process request end-of-body is processRequestEndOfBody,
    #    process early hints response is processEarlyHintsResponse, process response is processResponse, process
    #    response consume body is processResponseConsumeBody, process response end-of-body is processResponseEndOfBody,
    #    task destination is taskDestination, and cross-origin isolated capability is crossOriginIsolatedCapability.
    let fetch_params = fetch_params_create(vm, request, timing_info)
    fetch_params_set_algorithms(fetch_params, (FetchAlgorithms*)algorithms)
    fetch_params_set_task_destination(fetch_params, task_destination)
    fetch_params_set_cross_origin_isolated_capability(fetch_params, cross_origin_isolated_capability)

    # 9. If request’s body is a byte sequence, then set request’s body to request’s body as a body.
    match fetch_request_body_const(request):
        .byte_buffer(buffer):
            fetch_request_set_body(request, .fetch_body(fetch_byte_sequence_as_body(realm, ak.byte_buffer_bytes_const(&buffer))))
        _: _ = 0

    # 10. If all of the following conditions are true:
    let url = fetch_request_url(request)
    let scheme = web_url.url_scheme(&url)
    let mode = fetch_request_mode(request)
    let method = fetch_request_method(request)
    if (fetch_is_http_or_https_scheme(ak.string_bytes_as_string_view(&scheme)) and (mode == FetchRequestMode.same_origin or mode == FetchRequestMode.cors or mode == FetchRequestMode.no_cors) and fetch_fetching_client_global_is_window(request) and ak.byte_string_equals_ignoring_ascii_case(&method, ak.sv("GET")) and (not fetch_request_unsafe_request(request) or http_header_list_is_empty(fetch_request_header_list(request)))):
        # - request’s URL’s scheme is an HTTP(S) scheme
        # - request’s mode is "same-origin", "cors", or "no-cors"
        # - request’s client is not null, and request’s client’s global object is a Window object
        # - request’s method is `GET`
        # - request’s unsafe-request flag is not set or request’s header list is empty
        let client = fetch_request_client(request) else trap("VERIFY: the request's client")

        # 1. Assert: request’s origin is same origin with request’s client’s origin.
        match fetch_request_origin(request):
            .origin(origin): assert(web_url.origin_is_same_origin(&origin, html_environment_settings_object_origin(client)))
            _: trap("VERIFY: request.origin().has<URL::Origin>()")

        # 2. Let onPreloadedResponseAvailable be an algorithm that runs the following step given a response
        #    response: set fetchParams’s preloaded response candidate to response.
        let captures = new FetchFetchingFetchOnPreloadedResponseAvailable(fetch_params = fetch_params) else trap("out of memory")
        let on_preloaded_response_available = gc.create_function1(realm_heap(realm), ak.Function1[FetchResponse*, unit](call = FetchFetchingFetchOnPreloadedResponseAvailable.call, context = captures))

        # 3. Let foundPreloadedResource be the result of invoking consume a preloaded resource for request’s
        #    window, given request’s URL, request’s destination, request’s mode, request’s credentials mode,
        #    request’s integrity metadata, and onPreloadedResponseAvailable.
        let window = as_html_window((gc.Cell*)html_environment_settings_object_global_object(client))
        let found_preloaded_resource = html_consume_a_preloaded_resource(
            window,
            fetch_request_url(request),
            fetch_request_destination(request),
            fetch_request_mode(request),
            fetch_request_credentials_mode(request),
            fetch_request_integrity_metadata(request),
            on_preloaded_response_available)

        # 4. If foundPreloadedResource is true and fetchParams’s preloaded response candidate is null, then set
        #    fetchParams’s preloaded response candidate to "pending".
        if found_preloaded_resource and fetch_params_preloaded_response_candidate_const(fetch_params) == .empty:
            fetch_params_set_preloaded_response_candidate(fetch_params, .fetch_params_preloaded_response_candidate_pending_tag(FetchParamsPreloadedResponseCandidatePendingTag()))

    # 11. If request’s header list does not contain `Accept`, then:
    if not http_header_list_contains(fetch_request_header_list(request), ak.sv("Accept")):
        # 1. Let value be `*/*`.
        var value = ak.sv("*/*")

        # 2. If request’s initiator is "prefetch", then set value to the document `Accept` header value.
        if fetch_request_initiator(request) == FetchRequestInitiator.prefetch:
            value = ak.sv(fetch_fetching_document_accept_header_value)

        # 3. Otherwise, the user agent should set value to the first matching statement, if any, switching on request’s destination:
        elif let destination = fetch_request_destination(request):
            match destination:
                # -> "document"
                # -> "frame"
                # -> "iframe"
                .document, .frame, .i_frame:
                    # the document `Accept` header value
                    value = ak.sv(fetch_fetching_document_accept_header_value)
                # -> "image"
                .image:
                    # `image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5`
                    # AD-HOC: The spec default omits AVIF and WebP, which causes CDNs that perform format negotiation to
                    #         potentially fall back to non alpha-preserving formats.
                    #         Spec issue: https://github.com/whatwg/fetch/issues/1740
                    value = ak.sv("image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5")
                # -> "json"
                .json:
                    # `application/json,*/*;q=0.5`
                    value = ak.sv("application/json,*/*;q=0.5")
                # -> "style"
                .style:
                    # `text/css,*/*;q=0.1`
                    value = ak.sv("text/css,*/*;q=0.1")
                _: _ = 0

        # 4. Append (`Accept`, value) to request’s header list.
        let header = http_header_isomorphic_encode(ak.sv("Accept"), value)
        http_header_list_append(fetch_request_header_list(request), header)

    # 12. If request’s header list does not contain `Accept-Language`, then user agents should append
    #     (`Accept-Language, an appropriate header value) to request’s header list.
    if not http_header_list_contains(fetch_request_header_list(request), ak.sv("Accept-Language")):
        var accept_language = ak.string_builder_new()
        ak.string_builder_join_string(&accept_language, ak.sv(","), ak.vector_span_const(resource_loader_preferred_languages()))

        let header = http_header_isomorphic_encode(ak.sv("Accept-Language"), ak.string_builder_string_view(&accept_language))
        http_header_list_append(fetch_request_header_list(request), header)

    # 13. If request’s internal priority is null, then use request’s priority, initiator, destination, and
    #     render-blocking in an implementation-defined manner to set request’s internal priority to an
    #     implementation-defined object.
    # NOTE: The user-agent-defined object could encompass stream weight and dependency for HTTP/2, and equivalent
    #       information used to prioritize dispatch and processing of HTTP/1 fetches.

    # 14. If request is a subresource request, then:
    if fetch_request_is_subresource_request(request):
        # 1. Let record be a new fetch record whose request is request and controller is fetchParams’s controller.
        let record = fetch_record_create_fetch_controller(vm, request, fetch_params_controller(fetch_params))

        # 2. Append record to request’s client’s fetch group’s fetch records.
        let client = fetch_request_client(request) else trap("VERIFY: a subresource request's client")
        ak.intrusive_list_append(html_environment_settings_object_fetch_group(client), record, memory.offset_of(FetchRecord, m_list_node))

    # 15. Run main fetch given fetchParams.
    _ = fetch_fetching_main_fetch(realm, fetch_params)

    # 16. Return fetchParams’s controller.
    return fetch_params_controller(fetch_params)

## `request.client() && is<HTML::Window>(request.client()->global_object())`, a condition of fetch's step 10.
func fetch_fetching_client_global_is_window(request: FetchRequest*) -> bool:
    let client = fetch_request_client(request) else return false
    return is_html_window((gc.Cell*)html_environment_settings_object_global_object(client))

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:159 Web::Fetch::Fetching::fetch with UseParallelQueue::Yes
## (region r28's entry point for perform_navigation_params_fetch).
pub func fetch_fetching_fetch_use_parallel_queue(realm: Realm*, request: FetchRequest*, algorithms: const FetchAlgorithms*) -> FetchController*:
    return fetch_fetching_fetch(realm, request, algorithms, FetchUseParallelQueue.yes)

## Ported from Libraries/LibWeb/Fetch/Fetching/Fetching.cpp:336 Web::Fetch::Fetching::populate_request_from_client.
# https://fetch.spec.whatwg.org/#populate-request-from-client
pub func fetch_fetching_populate_request_from_client(realm: Realm*, request: FetchRequest*):
    let heap = realm_heap(realm)

    # 1. If request’s traversable for user prompts is "client":
    match fetch_request_traversable_for_user_prompts(request):
        .fetch_request_traversable_for_user_prompts(traversable_for_user_prompts) if traversable_for_user_prompts == FetchRequestTraversableForUserPrompts.client:
            # 1. Set request’s traversable for user prompts to "no-traversable".
            fetch_request_set_traversable_for_user_prompts(request, .fetch_request_traversable_for_user_prompts(FetchRequestTraversableForUserPrompts.no_traversable))

            # 2. If request’s client is non-null:
            if let client = fetch_request_client(request):
                # 1. Let global be request’s client’s global object.
                let global = html_environment_settings_object_global_object(client)

                # 2. If global is a Window object and global’s navigable is not null, then set request’s traversable for
                #    user prompts to global’s navigable’s traversable navigable.
                if let window = as_if_html_window((gc.Cell*)global):
                    if let navigable = html_window_navigable(window):
                        fetch_request_set_traversable_for_user_prompts(request, .html_traversable_navigable(html_navigable_traversable_navigable(navigable)))
        _: _ = 0

    # 2. If request’s origin is "client":
    match fetch_request_origin(request):
        .fetch_request_origin(origin) if origin == FetchRequestOrigin.client:
            # 1. Assert: request’s client is non-null.
            let client = fetch_request_client(request) else trap("VERIFY: request.client()")

            # 2. Set request’s origin to request’s client’s origin.
            fetch_request_set_origin(request, .origin(html_environment_settings_object_origin(client)))
        _: _ = 0

    # 3. If request’s policy container is "client":
    match fetch_request_policy_container(request):
        .fetch_request_policy_container(policy_container) if policy_container == FetchRequestPolicyContainer.client:
            # 1. If request’s client is non-null, then set request’s policy container to a clone of request’s client’s
            #    policy container.
            if let client = fetch_request_client(request):
                fetch_request_set_policy_container(request, .html_policy_container(html_policy_container_clone(html_environment_settings_object_policy_container(client), heap)))
            # 2. Otherwise, set request’s policy container to a new policy container.
            else:
                fetch_request_set_policy_container(request, .html_policy_container(heap_allocate_html_policy_container(heap)))
        _: _ = 0