dymokomi/luce-browser-engine
Source of release 0.6.1
0.6.1 / src / web / loader / tests_loader_6.lucb
#==============================================================================================
#
# tests_loader_6 - the non-blocking in-process client: Core::Notifier and the loop's
# readiness wait, a response dripping while timers fire, keep-alive reuse, concurrency and
# the per-host limit, a streamed gzip body, cancellation, timeouts, connection errors,
# TLS, preconnects, and a load of concurrent requests
#
# DESCRIPTION:
# The http(s) tests run on tests_loader_5.lucb's server, through the in-process client
# and the loader tests' event loop (LoaderTestEventLoop: deferred invocations, then the
# notifiers whose descriptors are ready, then the timers that are due). Each log checks
# the delivery order LibRequests gives WebContent: the headers, the body chunks, the
# end, once. Redirects through Fetch over the new client are tests_fetching_3's.
#
#==============================================================================================
# mark: Helpers =================================================================================
## Runs the loader tests' loop for `milliseconds`, whatever is pending.
func net_test_spin(event_loop: LoaderTestEventLoop*, milliseconds: u64):
let end = loader_test_now() + milliseconds * 1000000
while loader_test_now() < end:
while not ak.vector_is_empty(&event_loop.deferred):
_ = event_loop.pump(CoreEventLoopPumpMode.dont_wait_for_events)
if core_event_loop_notifiers_count(&event_loop.notifiers) == 0:
thread.sleep(1)
else:
var deadline = net.Deadline.at(end)
if let due = event_loop.next_due():
if due < end:
deadline = net.Deadline.at(due)
_ = core_event_loop_notifiers_wait(&event_loop.notifiers, deadline)
_ = core_event_loop_notifiers_dispatch(&event_loop.notifiers)
_ = event_loop.fire_timers()
## Waits (at most two seconds) until `counter` reaches `value`.
func net_test_wait_for(counter: (@u64)*, value: u64):
var waited = 0
while counter.load(.acquire) < value and waited < 200:
thread.sleep(10)
waited += 1
## Whether `log` heard one complete success with the body `text`, in order.
func net_test_succeeded(log: const LoaderTestSeamLog*, text: str) -> bool:
return log.finishes == 1 and log.network_error == none and log.status_code == 200 and not log.out_of_order and log.received(text)
## The client's connections: (all, idle).
func net_test_counts(client: const InProcessRequestClient*) -> (usize, usize):
return in_process_request_client_connection_counts(client)
## A repeating timer's ticks.
struct NetTestTicks:
var count: i32
static func tick(context: void*?):
let ticks = (NetTestTicks*)(context else trap("no ticks"))
ticks.count += 1
## A notifier's activations, and another notifier it disables when it fires.
struct NetTestActivations:
var count: i32
var disable: CoreNotifier*?
static func activated(context: void*?):
let activations = (NetTestActivations*)(context else trap("no activations"))
activations.count += 1
if let other = activations.disable:
core_notifier_set_enabled(other, false)
# mark: The event loop ==========================================================================
test "Core::Notifier: registered while enabled; the loop's wait activates the notifiers whose descriptors are ready":
let event_loop = install_test_event_loop()
var listener = net.Listener.bind(net.SocketAddress.loopback(0)) else trap("a listener")
defer listener.destroy()
var outgoing = net.Connection.connect(listener.address() else trap("an address")) else trap("a connection")
defer outgoing.destroy()
var incoming = listener.accept() else trap("accepted")
defer incoming.destroy()
let fd = (i32)(incoming.descriptor() else trap("a descriptor"))
var reads = NetTestActivations()
var writes = NetTestActivations()
let read = core_notifier_create(fd, CoreNotificationType.read)
read.on_activation = ak.Function0[unit](call = NetTestActivations.activated, context = &reads)
let write = core_notifier_create(fd, CoreNotificationType.write)
write.on_activation = ak.Function0[unit](call = NetTestActivations.activated, context = &writes)
assert(event_loop.notifier_registrations == 2 and core_event_loop_notifiers_count(&event_loop.notifiers) == 2)
# Writable at once, nothing to read: the write notifier only (both share the descriptor's slot).
let soon = net.Deadline.after(1000000000) else trap("a deadline")
assert(core_event_loop_notifiers_wait(&event_loop.notifiers, soon) == 1)
assert(core_event_loop_notifiers_dispatch(&event_loop.notifiers) == 1 and writes.count == 1 and reads.count == 0)
# A byte arrives: both.
_ = outgoing.write(b"x") else trap("written")
_ = incoming.wait(net.WaitInterest.read, soon) else trap("the byte arrives")
assert(core_event_loop_notifiers_wait(&event_loop.notifiers, soon) == 2)
_ = core_event_loop_notifiers_dispatch(&event_loop.notifiers)
assert(writes.count == 2 and reads.count == 1)
# Disabled: no longer watched; set_type re-registers; close unregisters for good.
core_notifier_set_enabled(write, false)
assert(event_loop.notifier_unregistrations == 1 and core_event_loop_notifiers_count(&event_loop.notifiers) == 1)
core_notifier_set_type(read, CoreNotificationType.write)
assert(event_loop.notifier_registrations == 3 and core_notifier_type(read) == CoreNotificationType.write)
core_notifier_set_type(read, CoreNotificationType.read)
# A notifier unregistered by an activation before it in the same dispatch is not activated.
reads.disable = write
core_notifier_set_enabled(write, true)
_ = core_event_loop_notifiers_wait(&event_loop.notifiers, soon)
_ = core_event_loop_notifiers_dispatch(&event_loop.notifiers)
assert(reads.count == 2 and writes.count == 2)
core_notifier_close(read)
assert(core_notifier_fd(read) == -1 and not core_notifier_is_enabled(read))
core_notifier_set_enabled(read, true)
assert(core_event_loop_notifiers_count(&event_loop.notifiers) == 0)
# Nothing registered: the wait answers at once.
assert(core_event_loop_notifiers_wait(&event_loop.notifiers, net.Deadline()) == 0)
core_event_loop_installed = none
# mark: Requests on the event loop ==============================================================
test "the in-process client: a response that drips arrives in chunks while timers keep firing; keep-alive reuse":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let client = in_process_request_client_create(vm_heap(vm))
let server = net_test_server_start()
# The body comes ten bytes every 30 ms; a 5 ms timer fires all along.
var ticks = NetTestTicks()
let timer = core_timer_create_repeating(5, ak.Function0[unit](call = NetTestTicks.tick, context = &ticks))
core_timer_start(timer)
let (drip, _) = net_test_get(client, server, "/drip")
loader_test_drain(&event_loop)
core_timer_stop(timer)
assert(net_test_succeeded(drip, "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"))
assert(drip.chunks >= 5 and ticks.count >= 20 and drip.total_size == 100)
# The connection is kept: the next requests reuse it, one after another.
for _ in 0..<5:
let (again, _) = net_test_get(client, server, "/id")
loader_test_drain(&event_loop)
assert(net_test_succeeded(again, "1"))
assert(server.accepted.load(.acquire) == 1)
let (all, idle) = net_test_counts(client)
assert(all == 1 and idle == 1)
# A response that closes the connection without saying so: the next request gets a new connection.
let (closing, _) = net_test_get(client, server, "/close")
loader_test_drain(&event_loop)
assert(net_test_succeeded(closing, "close"))
let (fresh, _) = net_test_get(client, server, "/id")
loader_test_drain(&event_loop)
assert(net_test_succeeded(fresh, "2") and server.accepted.load(.acquire) == 2)
in_process_request_client_close_idle_connections(client)
net_test_server_stop(server)
test "the in-process client: concurrent requests, the per-host limit, a gzip body streamed in chunks":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let client = in_process_request_client_create(vm_heap(vm))
let server = net_test_server_start()
# Twenty at once, no limit: twenty connections, served together.
var logs: ak.Vector[LoaderTestSeamLog*]
for _ in 0..<20:
let (log, _) = net_test_get(client, server, "/keep")
ak.vector_append_t(&logs, log)
loader_test_drain(&event_loop)
for log in ak.vector_span_const(&logs):
assert(net_test_succeeded(log, "keep"))
assert(server.accepted.load(.acquire) == 20)
in_process_request_client_close_idle_connections(client)
# At most two connections to the origin: the others wait and take them in turn.
client.m_max_host_connections = 2
ak.vector_clear(&logs)
for _ in 0..<10:
let (log, _) = net_test_get(client, server, "/id")
ak.vector_append_t(&logs, log)
let (all, _) = net_test_counts(client)
assert(all == 0)
loader_test_drain(&event_loop)
for log in ak.vector_span_const(&logs):
assert(log.finishes == 1 and log.network_error == none and not log.out_of_order)
assert(log.received("21") or log.received("22"))
assert(server.accepted.load(.acquire) == 22)
# A gzip body sent chunked, a growing piece at a time: decoded as it comes, delivered in chunks.
let (gzip, _) = net_test_get(client, server, "/chunked-gzip")
loader_test_drain(&event_loop)
let text = net_test_gzip_text()
defer free(text) in memory.heap
assert(gzip.finishes == 1 and gzip.network_error == none and not gzip.out_of_order and gzip.chunks >= 2)
assert(gzip.total_size == (u64)text.length and ak.compare_bytes(ak.byte_buffer_bytes_const(&gzip.data), text) == 0)
in_process_request_client_close_idle_connections(client)
net_test_server_stop(server)
test "the in-process client: stopping a request in flight closes its connection; a waiting one leaves the queue; timeouts":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let client = in_process_request_client_create(vm_heap(vm))
let server = net_test_server_start()
client.m_max_host_connections = 1
# A request the server never answers, and one waiting behind it for the only connection.
let (stalled, stalled_request) = net_test_get(client, server, "/hang")
let (queued, queued_request) = net_test_get(client, server, "/keep")
net_test_spin(&event_loop, 100)
assert(stalled.headers == none and queued.headers == none)
assert(requests_request_stop(queued_request))
assert(requests_request_stop(stalled_request))
assert(not requests_request_stop(stalled_request))
net_test_wait_for(&server.abandoned, 1)
assert(server.abandoned.load(.acquire) == 1)
net_test_spin(&event_loop, 50)
assert(stalled.finishes == 0 and queued.finishes == 0)
let (all, _) = net_test_counts(client)
assert(all == 0)
# A finished request cannot be stopped.
let (done, done_request) = net_test_get(client, server, "/keep")
loader_test_drain(&event_loop)
assert(net_test_succeeded(done, "keep") and not requests_request_stop(done_request))
# The request's time limit: TimeoutReached, and the connection is closed.
client.m_timeout_nanoseconds = 200000000
let (timed_out, _) = net_test_get(client, server, "/hang")
loader_test_drain(&event_loop)
assert(timed_out.finishes == 1 and timed_out.network_error == RequestsNetworkError.timeout_reached and timed_out.headers == none)
net_test_wait_for(&server.abandoned, 2)
assert(server.abandoned.load(.acquire) == 2)
in_process_request_client_close_idle_connections(client)
net_test_server_stop(server)
test "the in-process client: names resolved off the loop, unknown hosts, refused connections, preconnects":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let client = in_process_request_client_create(vm_heap(vm))
let server = net_test_server_start()
# localhost is resolved on the lookup's thread (its addresses tried in order: ::1 is refused, 127.0.0.1 answers).
let (named, _) = net_test_get(client, server, "/keep", "localhost")
loader_test_drain(&event_loop)
assert(net_test_succeeded(named, "keep"))
let (unknown, _) = loader_test_start(client, "http://no-such-host.invalid/")
loader_test_drain(&event_loop)
assert(unknown.finishes == 1 and unknown.headers == none)
assert(unknown.network_error == RequestsNetworkError.unable_to_resolve_host or unknown.network_error == RequestsNetworkError.unable_to_connect)
var closed = net.Listener.bind(net.SocketAddress.loopback(0)) else trap("a listener")
let closed_port = (closed.address() else trap("an address")).port
closed.destroy()
var refused_url = ak.string_builder_new()
ak.string_builder_append(&refused_url, "http://127.0.0.1:")
loader_test_append_number(&refused_url, (i64)closed_port)
ak.string_builder_append(&refused_url, "/")
let (refused, _) = loader_test_start(client, ak.string_view_text(ak.string_builder_string_view(&refused_url)))
loader_test_drain(&event_loop)
assert(refused.finishes == 1 and refused.network_error == RequestsNetworkError.unable_to_connect)
in_process_request_client_close_idle_connections(client)
# ensure_connection (CreateConnection): the connection is opened ahead and the request takes it.
let id_url = net_test_url(server, "/id")
requests_request_client_ensure_connection(&client.client, loader_test_url(ak.string_view_text(ak.byte_string_view(&id_url))), RequestServerCacheLevel.create_connection)
net_test_wait_for(&server.accepted, 2)
net_test_spin(&event_loop, 50)
let (all, idle) = net_test_counts(client)
assert(all == 1 and idle == 1)
let (preconnected, _) = net_test_get(client, server, "/id")
loader_test_drain(&event_loop)
assert(net_test_succeeded(preconnected, "2") and server.accepted.load(.acquire) == 2)
in_process_request_client_close_idle_connections(client)
net_test_server_stop(server)
test "the in-process client: https: on a TLS server with a pinned issuer, kept alive; an untrusted certificate fails":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let server = net_test_server_start(true)
let client = in_process_request_client_create(vm_heap(vm))
client.m_trust = tls_stream.Trust.pinned_issuer(server.issuer_x, server.issuer_y) else trap("a pinned issuer")
let (secure, _) = net_test_get(client, server, "/keep", "localhost")
loader_test_drain(&event_loop)
assert(net_test_succeeded(secure, "keep"))
let (big, _) = net_test_get(client, server, "/big", "localhost")
loader_test_drain(&event_loop)
assert(big.finishes == 1 and big.network_error == none and not big.out_of_order and ak.byte_buffer_size(&big.data) == net_test_big_size)
let body = ak.byte_buffer_bytes_const(&big.data)
for index in 0..<body.length:
assert(body[index] == net_test_big_byte(index))
assert(server.accepted.load(.acquire) == 1)
in_process_request_client_close_idle_connections(client)
# The public roots do not include the test certificate's issuer.
let untrusting = in_process_request_client_create(vm_heap(vm))
let (refused, _) = net_test_get(untrusting, server, "/keep", "localhost")
loader_test_drain(&event_loop)
assert(refused.finishes == 1 and refused.network_error == RequestsNetworkError.ssl_handshake_failed and refused.headers == none)
net_test_server_stop(server)
test "the in-process client under load: hundreds of concurrent requests, with and without a per-host limit":
let vm = web_test_create_vm()
defer web_test_destroy_vm(vm)
# The VM's heap is current, as the engine runs: what the requests deliver is managed, kept by the loop's queue.
let allocator: memory.Allocator = vm_heap(vm)
with allocator:
var event_loop = LoaderTestEventLoop()
loader_test_install_event_loop(&event_loop)
defer loader_test_remove_event_loop()
let server = net_test_server_start()
let client = in_process_request_client_create(vm_heap(vm))
let text = net_test_gzip_text()
defer free(text) in memory.heap
for round in 0..<4:
client.m_max_host_connections = 0 if round % 2 == 0 else 6
in_process_request_client_close_idle_connections(client)
let accepted_before = server.accepted.load(.acquire)
var logs: ak.Vector[LoaderTestSeamLog*]
var paths: ak.Vector[i32]
for index in 0..<150:
let kind = (i32)(index % 15)
var path = "/keep"
if kind == 1:
path = "/chunked-gzip"
elif kind == 2:
path = "/drip"
elif kind == 3 and index % 45 == 3:
path = "/big"
let (log, _) = net_test_get(client, server, path)
ak.vector_append_t(&logs, log)
ak.vector_append_t(&paths, kind if path != "/keep" else 0)
loader_test_drain(&event_loop)
for index in 0..<ak.vector_size(&logs):
let log = *ak.vector_at_const(&logs, index)
assert(log.finishes == 1 and log.network_error == none and not log.out_of_order and log.status_code == 200)
match *ak.vector_at_const(&paths, index):
1: assert(ak.compare_bytes(ak.byte_buffer_bytes_const(&log.data), text) == 0)
2: assert(ak.byte_buffer_size(&log.data) == 100)
3: assert(ak.byte_buffer_size(&log.data) == net_test_big_size)
_: assert(log.received("keep"))
# With the limit, six connections carry all of them.
if round % 2 == 1:
assert(server.accepted.load(.acquire) - accepted_before <= 6)
in_process_request_client_close_idle_connections(client)
net_test_server_stop(server)