dymokomi/luce-browser-engine
Source of release 0.6.0
0.6.0 / src / web / fetch / fetching / tests_fetching_3.lucb
#==============================================================================================
#
# tests_fetching_3 - fetches over HTTP on a loopback server: no-cors and CORS responses, the
# CORS-preflight fetch, redirects, network errors, the request's headers
#
# DESCRIPTION:
# Region p2c's HTTP scenarios: a server on 127.0.0.1 (luce-std net, its own thread, the C
# heap) answers each request with the canned response its route names ("METHOD path") and
# records every request head it received; the fetches run as tests_fetching_2's, from a
# test page whose about:blank document has an opaque origin (so CORS answers `null` or
# `*`). They cover HTTP fetch, HTTP-network-or-cache fetch (the headers it adds, the cache
# partition), the network fetch through the ResourceLoader, the CORS and TAO checks, the
# CORS-preflight fetch, HTTP-redirect fetch by redirect mode, and failed connections.
#
#==============================================================================================
# mark: A loopback server with routes ===========================================================
## A route of the server: the request line's method and target, and the canned response.
struct FetchingTestRoute:
var key: str
var response: const u8[]
## The routes of the HTTP scenarios.
let fetching_test_routes: FetchingTestRoute[12] = [
FetchingTestRoute(key = "GET /image", response = b"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 6\r\nX-Hidden: no\r\n\r\npixels"),
FetchingTestRoute(key = "GET /cors", response = b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 4\r\nAccess-Control-Allow-Origin: *\r\nAccess-Control-Expose-Headers: X-Exposed\r\nX-Exposed: yes\r\nX-Hidden: no\r\nTiming-Allow-Origin: *\r\n\r\ncors"),
FetchingTestRoute(key = "GET /no-cors-headers", response = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nno"),
FetchingTestRoute(key = "OPTIONS /put", response = b"HTTP/1.1 204 No Content\r\nAccess-Control-Allow-Origin: null\r\nAccess-Control-Allow-Methods: PUT\r\nAccess-Control-Allow-Headers: X-Custom\r\nContent-Length: 0\r\n\r\n"),
FetchingTestRoute(key = "PUT /put", response = b"HTTP/1.1 200 OK\r\nAccess-Control-Allow-Origin: null\r\nContent-Length: 3\r\n\r\nput"),
FetchingTestRoute(key = "OPTIONS /put-refused", response = b"HTTP/1.1 204 No Content\r\nAccess-Control-Allow-Origin: null\r\nContent-Length: 0\r\n\r\n"),
FetchingTestRoute(key = "GET /redirect", response = b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\n\r\n"),
FetchingTestRoute(key = "GET /final", response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nfinal"),
FetchingTestRoute(key = "GET /cors-redirect", response = b"HTTP/1.1 302 Found\r\nLocation: /final\r\nAccess-Control-Allow-Origin: *\r\nContent-Length: 0\r\n\r\n"),
FetchingTestRoute(key = "GET /redirect-to-data", response = b"HTTP/1.1 302 Found\r\nLocation: data:,x\r\nContent-Length: 0\r\n\r\n"),
FetchingTestRoute(key = "GET /not-found", response = b"HTTP/1.1 404 Not Found\r\nContent-Length: 4\r\n\r\nnope"),
FetchingTestRoute(key = "GET /quit", response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"),
]
## The server's state, in the C heap: its listener, and the heads of the requests it received.
struct FetchingTestServer:
var listener: net.Listener
var port: u16
var request_count: usize
var requests: u8[16][2048]
var request_lengths: usize[16]
## The canned response of a request line's `method target`, or a 500.
func fetching_test_server_response(key: const u8[]) -> const u8[]:
for route in fetching_test_routes:
if key.length == route.key.length and ak.compare_bytes(key, route.key.bytes) == 0:
return route.response
return b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n"
## The server thread: serves connections until `GET /quit`.
func fetching_test_server_main(context: void*?):
let server = (FetchingTestServer*)(context else trap("no server"))
var quit = false
while not quit:
var connection = server.listener.accept() catch:
return
var buffer: u8[4096] = ---
var used: usize = 0
var head_end: usize? = none
while head_end == none and used < buffer.length:
let count = connection.read(buffer[used..]) catch:
break
if count == 0:
break
used += count
head_end = net.http_head_end(buffer[..<used])
if let end = head_end:
let body_length = loader_test_server_content_length(buffer[..<end])
while used < end + body_length and used < buffer.length:
let count = connection.read(buffer[used..]) catch:
break
if count == 0:
break
used += count
if server.request_count < 16:
let length = end if end < 2048 else 2048
memory.copy(server.requests[server.request_count], buffer, length)
server.request_lengths[server.request_count] = length
server.request_count += 1
# The key: the request line up to the second space.
var first: usize = 0
while first < end and buffer[first] != ' ':
first += 1
var second = first + 1
while second < end and buffer[second] != ' ':
second += 1
let key = buffer[..<second]
if key.length == 9 and ak.compare_bytes(key, b"GET /quit") == 0:
quit = true
_ = connection.write(fetching_test_server_response(key)) catch: recover 0
connection.shutdown() catch:
_ = 0
connection.destroy()
## A server on 127.0.0.1 running on its own thread.
func fetching_test_server_start() -> (FetchingTestServer*, thread.Handle):
with memory.heap:
let listener = net.Listener.bind(net.SocketAddress.loopback(0)) else trap("the server's listener")
let address = listener.address() else trap("the server's address")
let server = new FetchingTestServer(listener = listener, port = address.port) else trap("out of memory")
let handle = thread.spawn(fetching_test_server_main, server, name = "p2c-http-server") else trap("the server's thread")
return (server, handle)
## The end of a server: `GET /quit` sent, its thread joined, its listener closed.
func fetching_test_server_stop(server: FetchingTestServer*, handle: thread.Handle):
with memory.heap:
var connection = net.Connection.connect(net.SocketAddress.loopback(server.port)) else trap("the server accepts")
_ = connection.write(b"GET /quit HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") catch: recover 0
var reply: u8[256] = ---
_ = connection.read(reply) catch: recover 0
connection.destroy()
var joined = handle
joined.join() else trap("the server's thread joins")
server.listener.destroy()
free(server) in memory.heap
## `http://127.0.0.1:<port><path>`.
func fetching_test_server_url(server: const FetchingTestServer*, path: str) -> ak.ByteString:
var builder = ak.string_builder_new()
ak.string_builder_append(&builder, "http://127.0.0.1:")
loader_test_append_number(&builder, (i64)server.port)
ak.string_builder_append(&builder, path)
return ak.string_builder_to_byte_string(&builder)
## The head of the `index`th request the server received.
func fetching_test_server_request(server: const FetchingTestServer*, index: usize) -> ak.StringView:
assert(index < server.request_count)
return ak.string_view_from_bytes(server.requests[index][..<server.request_lengths[index]])
## Whether the `index`th request's head has the line `line` (`Name: value`, or the request line).
func fetching_test_server_request_has(server: const FetchingTestServer*, index: usize, line: str) -> bool:
let head = fetching_test_server_request(server, index)
var needle = ak.string_builder_new()
ak.string_builder_append(&needle, line)
ak.string_builder_append(&needle, "\r\n")
return ak.string_view_find_string_view(&head, ak.string_builder_string_view(&needle)) != none
## A request of the page's client for `path` on the server.
func fetching_test_server_request_for(page: const FetchingTestPage*, server: const FetchingTestServer*, path: str, destination: FetchRequestDestination?, mode: FetchRequestMode) -> FetchRequest*:
let url = fetching_test_server_url(server, path)
return fetching_test_request(page, ak.string_view_text(ak.byte_string_view(&url)), destination, mode)
# mark: HTTP scenarios ==========================================================================
test "fetch over HTTP: no-cors (opaque) and CORS responses, their headers, the request's headers, the CORS check":
var event_loop = FetchingTestEventLoop()
fetching_test_begin(&event_loop)
defer fetching_test_end()
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
_ = web_test_install_agent(vm)
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
let page = fetching_test_page(vm, &event_loop)
let (server, handle) = fetching_test_server_start()
# An image from another origin, no-cors: an opaque filtered response (no status, headers or body), its
# internal response's bytes read for process response consume body.
let image = fetching_test_server_request_for(&page, server, "/image", FetchRequestDestination.image, FetchRequestMode.no_cors)
let (image_log, _) = fetching_test_fetch(&page, image)
fetching_test_run(&event_loop)
assert(image_log.responses == 1 and image_log.consumed == 1 and image_log.ends_of_body == 1)
let opaque = image_log.the_response()
assert(fetch_response_type(opaque) == FetchResponseType.opaque and fetch_response_status(opaque) == 0)
assert(http_header_list_is_empty(fetch_response_header_list(opaque)) and fetch_response_body(opaque) == none)
assert(image_log.received("pixels"))
let internal = fetch_filtered_response_internal_response(opaque)
assert(fetch_response_status(internal) == 200 and fetching_test_header_is(fetch_response_header_list(internal), "X-Hidden", "no"))
assert(fetch_request_redirect_count(image) == 0)
# The head sent: the default Accept for images, Accept-Language, the Fetch metadata headers (127.0.0.1 is
# potentially trustworthy), the default User-Agent.
assert(server.request_count == 1)
assert(fetching_test_server_request_has(server, 0, "GET /image HTTP/1.1"))
assert(fetching_test_server_request_has(server, 0, "Accept: image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5"))
assert(fetching_test_server_request_has(server, 0, "Accept-Language: en-US"))
assert(fetching_test_server_request_has(server, 0, "Sec-Fetch-Dest: image"))
assert(fetching_test_server_request_has(server, 0, "Sec-Fetch-Mode: no-cors"))
assert(fetching_test_server_request_has(server, 0, "Sec-Fetch-Site: cross-site"))
let user_agent = fetch_default_user_agent_value()
var user_agent_line = ak.string_builder_new()
ak.string_builder_append(&user_agent_line, "User-Agent: ")
ak.string_builder_append_string_view(&user_agent_line, ak.byte_string_view(&user_agent))
assert(fetching_test_server_request_has(server, 0, ak.string_view_text(ak.string_builder_string_view(&user_agent_line))))
# A CORS request whose response allows any origin: a CORS filtered response exposing the safelisted and the
# exposed headers only; the TAO check passes.
let cors = fetching_test_server_request_for(&page, server, "/cors", none, FetchRequestMode.cors)
let (cors_log, _) = fetching_test_fetch(&page, cors)
fetching_test_run(&event_loop)
let cors_response = cors_log.the_response()
assert(fetch_response_type(cors_response) == FetchResponseType.cors and fetch_response_status(cors_response) == 200)
assert(fetching_test_header_is(fetch_response_header_list(cors_response), "X-Exposed", "yes"))
assert(fetching_test_header_is(fetch_response_header_list(cors_response), "Content-Type", "text/plain"))
assert(http_header_list_get(fetch_response_header_list(cors_response), ak.sv("X-Hidden")) == none)
assert(cors_log.received("cors") and not fetch_request_timing_allow_failed(cors))
assert(fetch_request_response_tainting(cors) == FetchRequestResponseTainting.cors)
assert(fetching_test_server_request_has(server, 1, "Origin: null"))
assert(fetching_test_server_request_has(server, 1, "Sec-Fetch-Mode: cors"))
# Without Access-Control-Allow-Origin, the CORS check fails: a network error.
let refused = fetching_test_server_request_for(&page, server, "/no-cors-headers", none, FetchRequestMode.cors)
let (refused_log, _) = fetching_test_fetch(&page, refused)
fetching_test_run(&event_loop)
assert(fetching_test_network_error_is(refused_log.the_response(), "Request with 'cors' response tainting failed CORS check"))
assert(refused_log.consumed == 1 and refused_log.consumed_null)
# A 404 is a response (no network error).
let not_found = fetching_test_server_request_for(&page, server, "/not-found", FetchRequestDestination.image, FetchRequestMode.no_cors)
let (not_found_log, _) = fetching_test_fetch(&page, not_found)
fetching_test_run(&event_loop)
assert(fetch_response_status(fetch_filtered_response_internal_response(not_found_log.the_response())) == 404 and not_found_log.received("nope"))
# The HTTP cache partition of the page's network partition key exists (the memory cache itself is off).
assert(not fetch_fetching_http_memory_cache_enabled())
assert(ak.hash_map_size(&fetch_http_cache_the().m_cache) >= 1)
fetching_test_server_stop(server, handle)
# Nothing listens any more: a network error, the stream errored with a TypeError.
let gone = fetching_test_server_request_for(&page, server, "/image", FetchRequestDestination.image, FetchRequestMode.no_cors)
let (gone_log, _) = fetching_test_fetch(&page, gone)
fetching_test_run(&event_loop)
let gone_response = gone_log.the_response()
assert(fetch_response_is_network_error(gone_response))
let gone_message = fetch_response_network_error_message(gone_response) else trap("a message")
assert(ak.string_starts_with_bytes(&gone_message, ak.sv("Load failed: ")))
fetch_fetching_clear_http_memory_cache()
test "fetch over HTTP: the CORS-preflight fetch, then the main request; a preflight that refuses the method":
var event_loop = FetchingTestEventLoop()
fetching_test_begin(&event_loop)
defer fetching_test_end()
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
_ = web_test_install_agent(vm)
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
let page = fetching_test_page(vm, &event_loop)
let (server, handle) = fetching_test_server_start()
# A CORS PUT with a custom header (an unsafe request): an OPTIONS preflight naming them, then the PUT.
let put = fetching_test_server_request_for(&page, server, "/put", none, FetchRequestMode.cors)
fetch_request_set_method(put, ak.byte_string_from_str("PUT"))
fetch_request_set_unsafe_request(put, true)
http_header_list_append(fetch_request_header_list(put), http_header_create(ak.sv("X-Custom"), ak.sv("1")))
let (put_log, _) = fetching_test_fetch(&page, put)
fetching_test_run(&event_loop)
assert(put_log.responses == 1 and put_log.received("put"))
assert(fetch_response_type(put_log.the_response()) == FetchResponseType.cors)
assert(server.request_count == 2)
assert(fetching_test_server_request_has(server, 0, "OPTIONS /put HTTP/1.1"))
assert(fetching_test_server_request_has(server, 0, "Accept: */*"))
assert(fetching_test_server_request_has(server, 0, "Access-Control-Request-Method: PUT"))
assert(fetching_test_server_request_has(server, 0, "Access-Control-Request-Headers: x-custom"))
assert(fetching_test_server_request_has(server, 1, "PUT /put HTTP/1.1"))
assert(fetching_test_server_request_has(server, 1, "X-Custom: 1"))
assert(fetching_test_server_request_has(server, 1, "Content-Length: 0"))
# A preflight whose response allows no method: a network error, and no main request.
let refused = fetching_test_server_request_for(&page, server, "/put-refused", none, FetchRequestMode.cors)
fetch_request_set_method(refused, ak.byte_string_from_str("PUT"))
fetch_request_set_unsafe_request(refused, true)
let (refused_log, _) = fetching_test_fetch(&page, refused)
fetching_test_run(&event_loop)
assert(fetching_test_network_error_is(refused_log.the_response(), "Non-CORS-safelisted method 'PUT' not found in the CORS-preflight response's Access-Control-Allow-Methods header and there was no '*' entry. The header may be missing."))
assert(server.request_count == 3)
fetching_test_server_stop(server, handle)
fetch_fetching_clear_http_memory_cache()
test "fetch over HTTP: redirects followed, refused in error mode, to a scheme that is not HTTP(S)":
var event_loop = FetchingTestEventLoop()
fetching_test_begin(&event_loop)
defer fetching_test_end()
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
_ = web_test_install_agent(vm)
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
let page = fetching_test_page(vm, &event_loop)
let (server, handle) = fetching_test_server_start()
# Followed: the final response, whose URL list has both URLs; the request counted the redirect.
let followed = fetching_test_server_request_for(&page, server, "/redirect", FetchRequestDestination.image, FetchRequestMode.no_cors)
let (followed_log, _) = fetching_test_fetch(&page, followed)
fetching_test_run(&event_loop)
assert(followed_log.responses == 1 and followed_log.received("final"))
assert(fetch_request_redirect_count(followed) == 1)
let internal = fetch_filtered_response_internal_response(followed_log.the_response())
assert(ak.vector_size(fetch_response_url_list(internal)) == 2)
let final_url = fetch_response_url(internal) else trap("a URL")
let serialized = web_url.url_serialize(&final_url)
assert(ak.string_ends_with_bytes(&serialized, ak.sv("/final")))
assert(server.request_count == 2 and fetching_test_server_request_has(server, 1, "GET /final HTTP/1.1"))
# Redirect mode "error" (CORS, the redirect passing the CORS check): a network error.
let error_mode = fetching_test_server_request_for(&page, server, "/cors-redirect", none, FetchRequestMode.cors)
fetch_request_set_redirect_mode(error_mode, FetchRequestRedirectMode.error_)
let (error_log, _) = fetching_test_fetch(&page, error_mode)
fetching_test_run(&event_loop)
assert(fetching_test_network_error_is(error_log.the_response(), "Request with 'error' redirect mode received redirect response"))
# A redirect to data: is refused.
let to_data = fetching_test_server_request_for(&page, server, "/redirect-to-data", FetchRequestDestination.image, FetchRequestMode.no_cors)
let (to_data_log, _) = fetching_test_fetch(&page, to_data)
fetching_test_run(&event_loop)
assert(fetching_test_network_error_is(to_data_log.the_response(), "Request redirect URL must have HTTP or HTTPS scheme"))
fetching_test_server_stop(server, handle)
fetch_fetching_clear_http_memory_cache()