dymokomi/luce-browser-html
Source of release 0.5.1
0.5.1 / src / html_syntax / tests_html_tokenizer_scenarios.lucb
#==============================================================================================
#
# tests_html_tokenizer_scenarios - the input stream API, pinned to the donor
#
# DESCRIPTION:
# Streaming input, insertion points (document.write), nested insertion points, abort
# and a legacy encoding, each run as the local oracle's scenarios run the donor's
# HTMLTokenizer (.donors/luce-browser-tools/html_tokenizer_oracle, `oracle --scenarios`),
# compared with its output. dump_token is in tests_html_tokenizer_cases_1.
#
#==============================================================================================
# mark: The harness =============================================================================
# drain(): tokens until next_token answers none ("(none)") or after the EOF token.
func scenario_drain(builder: ak.StringBuilder*, tokenizer: HtmlTokenizer*, stop: HtmlTokenizerStopAtInsertionPoint):
while true:
let maybe_token = html_tokenizer_next_token(tokenizer, stop)
if maybe_token == none:
ak.string_builder_append(builder, "(none)\n")
return
var token = maybe_token else trap("unreachable")
dump_token(builder, &token)
if html_token_is_end_of_file(&token):
return
# drain_one(): one token of next_token(StopAtInsertionPoint::No).
func scenario_drain_one(builder: ak.StringBuilder*, tokenizer: HtmlTokenizer*):
let maybe_token = html_tokenizer_next_token(tokenizer)
if maybe_token == none:
ak.string_builder_append(builder, "(none)\n")
return
var token = maybe_token else trap("unreachable")
dump_token(builder, &token)
# show(): the unparsed input and the stream flags.
func scenario_show(builder: ak.StringBuilder*, tokenizer: HtmlTokenizer*):
ak.string_builder_append(builder, "unparsed=")
dump_escaped_string(builder, html_tokenizer_unparsed_input(tokenizer))
var buffer: u8[128]
ak.string_builder_append(builder, strings.format(buffer, f" defined={(i32)html_tokenizer_is_insertion_point_defined(tokenizer)} reached={(i32)html_tokenizer_is_insertion_point_reached(tokenizer)} closed={(i32)html_tokenizer_is_input_stream_closed(tokenizer)} eof_inserted={(i32)html_tokenizer_is_eof_inserted(tokenizer)} blocked={(i32)html_tokenizer_is_blocked(tokenizer)}\n") else trap("format"))
func check_scenario(builder: ak.StringBuilder*, expected: str):
let actual = ak.string_builder_string_view(builder)
if not ak.string_view_equals(&actual, expected):
print(f"expected:\n{expected}")
print(f"actual:\n{(str)ak.string_view_bytes(&actual)}")
trap("the scenario differs from the donor's")
# mark: Scenarios ===============================================================================
test "scenario: streaming input":
with memory.heap:
var builder = ak.string_builder_new()
var t: HtmlTokenizer
html_tokenizer_construct(&t)
html_tokenizer_append_to_input_stream(&t, ak.sv("<p cla"))
scenario_drain(&builder, &t, .no)
scenario_show(&builder, &t)
html_tokenizer_append_to_input_stream(&t, ak.sv("ss=x>a\r"))
scenario_drain(&builder, &t, .no)
html_tokenizer_append_to_input_stream(&t, ak.sv("\nb&no"))
scenario_drain(&builder, &t, .no)
html_tokenizer_append_to_input_stream(&t, ak.sv("t;c¬in"))
scenario_drain(&builder, &t, .no)
html_tokenizer_append_to_input_stream(&t, ak.sv("d\u{E9}"))
scenario_drain(&builder, &t, .no)
scenario_show(&builder, &t)
html_tokenizer_close_input_stream(&t)
scenario_drain(&builder, &t, .no)
scenario_show(&builder, &t)
check_scenario(&builder, """
(none)
unparsed= defined=0 reached=0 closed=0 eof_inserted=0 blocked=0
StartTag p self_closing=0 dup=0 attributes=1 [class=x 0:3-0:8 0:9-0:10] @0:1-0:10
Character U+0061 @0:12-0:0
(none)
Character U+000A @1:0-0:0
Character U+0062 @1:1-0:0
(none)
Character U+00AC @1:6-0:0
Character U+0063 @1:7-0:0
(none)
Character U+00AC @0:0-0:0
Character U+0069 @0:0-0:0
Character U+006E @0:0-0:0
Character U+0064 @0:0-0:0
Character U+00E9 @0:0-0:0
(none)
unparsed= defined=0 reached=0 closed=0 eof_inserted=0 blocked=0
EOF @0:0-0:0
unparsed= defined=0 reached=0 closed=1 eof_inserted=0 blocked=0
""")
test "scenario: insertion point":
with memory.heap:
var builder = ak.string_builder_new()
var t: HtmlTokenizer
html_tokenizer_construct(&t)
html_tokenizer_append_to_input_stream(&t, ak.sv("<a>12<b>tail</b>"))
scenario_drain_one(&builder, &t)
html_tokenizer_update_insertion_point(&t)
scenario_show(&builder, &t)
html_tokenizer_insert_input_at_insertion_point(&t, ak.sv("<i>w&</i>"))
scenario_show(&builder, &t)
scenario_drain(&builder, &t, .yes)
scenario_show(&builder, &t)
html_tokenizer_store_old_insertion_point(&t)
html_tokenizer_undefine_insertion_point(&t)
scenario_show(&builder, &t)
scenario_drain(&builder, &t, .no)
html_tokenizer_restore_old_insertion_point(&t)
scenario_show(&builder, &t)
html_tokenizer_close_input_stream(&t)
scenario_drain(&builder, &t, .no)
scenario_show(&builder, &t)
html_tokenizer_parser_did_run(&t)
scenario_show(&builder, &t)
check_scenario(&builder, """
StartTag a self_closing=0 dup=0 attributes=0 @0:1-0:2
unparsed=12<b>tail</b> defined=1 reached=1 closed=0 eof_inserted=0 blocked=0
unparsed=<i>w&</i>12<b>tail</b> defined=1 reached=0 closed=0 eof_inserted=0 blocked=0
StartTag i self_closing=0 dup=0 attributes=0 @0:4-0:5
Character U+0077 @0:7-0:0
Character U+0026 @0:11-0:0
EndTag i self_closing=0 dup=0 attributes=0 @0:13-0:14
(none)
unparsed=12<b>tail</b> defined=1 reached=1 closed=0 eof_inserted=0 blocked=0
unparsed=12<b>tail</b> defined=0 reached=0 closed=0 eof_inserted=0 blocked=0
Character U+0031 @0:16-0:0
Character U+0032 @0:17-0:0
StartTag b self_closing=0 dup=0 attributes=0 @0:18-0:19
Character U+0074 @0:21-0:0
Character U+0061 @0:22-0:0
Character U+0069 @0:23-0:0
Character U+006C @0:24-0:0
EndTag b self_closing=0 dup=0 attributes=0 @0:26-0:27
(none)
unparsed= defined=1 reached=1 closed=0 eof_inserted=0 blocked=0
EOF @0:28-0:28
unparsed= defined=1 reached=1 closed=1 eof_inserted=0 blocked=0
unparsed= defined=1 reached=1 closed=1 eof_inserted=0 blocked=0
""")
test "scenario: nested insertion points":
with memory.heap:
var builder = ak.string_builder_new()
var t: HtmlTokenizer
html_tokenizer_construct(&t)
html_tokenizer_append_to_input_stream(&t, ak.sv("x<y>z"))
scenario_drain_one(&builder, &t)
scenario_drain_one(&builder, &t)
html_tokenizer_update_insertion_point(&t)
html_tokenizer_store_old_insertion_point(&t)
html_tokenizer_insert_input_at_insertion_point(&t, ak.sv("AB"))
scenario_show(&builder, &t)
html_tokenizer_store_old_insertion_point(&t)
html_tokenizer_insert_input_at_insertion_point(&t, ak.sv("\u{20AC}\r"))
scenario_show(&builder, &t)
scenario_drain(&builder, &t, .yes)
html_tokenizer_insert_input_at_insertion_point(&t, ak.sv("\n"))
scenario_drain(&builder, &t, .yes)
html_tokenizer_restore_old_insertion_point(&t)
scenario_show(&builder, &t)
html_tokenizer_restore_old_insertion_point(&t)
scenario_show(&builder, &t)
html_tokenizer_insert_eof(&t)
html_tokenizer_set_blocked(&t, true)
scenario_show(&builder, &t)
html_tokenizer_undefine_insertion_point(&t)
scenario_drain(&builder, &t, .no)
check_scenario(&builder, """
Character U+0078 @0:1-0:0
StartTag y self_closing=0 dup=0 attributes=0 @0:2-0:3
unparsed=ABz defined=1 reached=0 closed=0 eof_inserted=0 blocked=0
unparsed=AB\u{20AC}\\rz defined=1 reached=0 closed=0 eof_inserted=0 blocked=0
Character U+0041 @0:5-0:0
Character U+0042 @0:6-0:0
Character U+20AC @0:7-0:0
(none)
Character U+000A @1:0-0:0
(none)
unparsed=z defined=1 reached=1 closed=0 eof_inserted=0 blocked=0
unparsed=z defined=1 reached=1 closed=0 eof_inserted=0 blocked=0
unparsed=z defined=1 reached=1 closed=1 eof_inserted=1 blocked=1
Character U+007A @1:1-0:0
EOF @1:1-1:1
""")
test "scenario: abort":
with memory.heap:
var builder = ak.string_builder_new()
var t: HtmlTokenizer
html_tokenizer_construct_string_view(&t, ak.sv("<a>b"), ak.byte_string_from_str("UTF-8"))
scenario_drain_one(&builder, &t)
html_tokenizer_abort(&t)
scenario_drain(&builder, &t, .no)
check_scenario(&builder, """
StartTag a self_closing=0 dup=0 attributes=0 @0:1-0:2
(none)
""")
test "scenario: windows-1252":
with memory.heap:
var builder = ak.string_builder_new()
var t: HtmlTokenizer
html_tokenizer_construct_string_view(&t, ak.string_view_from_bytes(b"<p title=\x93q\x94>\x80\xe9"), ak.byte_string_from_str("windows-1252"))
scenario_drain(&builder, &t, .no)
ak.string_builder_append(&builder, "source=")
dump_escaped_string(&builder, html_tokenizer_source(&t))
ak.string_builder_append(&builder, "\n")
check_scenario(&builder, """
StartTag p self_closing=0 dup=0 attributes=1 [title=\u{201C}q\u{201D} 0:3-0:8 0:9-0:12] @0:1-0:12
Character U+20AC @0:14-0:0
Character U+00E9 @0:15-0:0
EOF @0:15-0:15
source=<p title=\u{201C}q\u{201D}>\u{20AC}\u{E9}
""")