Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-css

Source of release 0.5.1

0.5.1 / src / css_syntax / tests_token_stream.lucb

#==============================================================================================
#
#   tests_token_stream - TokenStream (Tests/LibWeb/TestCSSTokenStream.cpp) and transactions
#
#   DESCRIPTION:
#       The two test cases of TestCSSTokenStream.cpp (basic, marks), then focused cases pinned
#       to the donor through a local probe: a stream of ComponentValues (dump_string,
#       discard_whitespace), StateTransaction rollback and commit through a child, and
#       tokens_since.
#
#==============================================================================================

## `Token::create_ident(name)` with the default (empty) source text.
func ident_token(name: str) -> Token:
    return token_create_ident(ak.fly_string_from_str(name), ak.string_new())

## Whether `token` is an ident token named `name`.
func is_ident_named(token: const Token*, name: str) -> bool:
    if not token_is(token, TokenType.ident):
        return false
    let ident = token_ident(token)
    let expected = ak.fly_string_from_str(name)
    return ak.fly_string_eq_fly_string(&ident, &expected)

# mark: TestCSSTokenStream.cpp ==================================================================

noinline func test_token_stream_basic():
    var tokens = ak.Vector[Token]()
    ak.vector_append_t(&tokens, ident_token("hello"))

    var stream = token_stream_create(tokens)
    assert(not token_stream_is_empty(&stream))
    assert(token_stream_has_next_token(&stream))
    assert(token_stream_remaining_token_count(&stream) == 1)

    # next_token() doesn't consume it
    let next = token_stream_next_token(&stream)
    assert(not token_stream_is_empty(&stream))
    assert(token_stream_has_next_token(&stream))
    assert(token_stream_remaining_token_count(&stream) == 1)
    # Check what the token is
    assert(token_is(&next, TokenType.ident))
    assert(is_ident_named(&next, "hello"))

    # consume_a_token() does consume it
    let consumed = token_stream_consume_a_token(&stream)
    assert(token_stream_is_empty(&stream))
    assert(not token_stream_has_next_token(&stream))
    assert(token_stream_remaining_token_count(&stream) == 0)
    # Check what the token is
    assert(token_is(&consumed, TokenType.ident))
    assert(is_ident_named(&consumed, "hello"))

    # Now, any further tokens should be EOF
    let after = token_stream_next_token(&stream)
    assert(token_is(&after, TokenType.end_of_file))
    let consumed_after = token_stream_consume_a_token(&stream)
    assert(token_is(&consumed_after, TokenType.end_of_file))

test "TestCSSTokenStream basic":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_token_stream_basic()

noinline func test_token_stream_marks():
    var tokens = ak.Vector[Token]()
    let names: str[7] = ["a", "b", "c", "d", "e", "f", "g"]
    for name in names:
        ak.vector_append_t(&tokens, ident_token(name))
    var stream = token_stream_create(tokens)

    token_stream_mark(&stream)  # 0

    assert(token_stream_remaining_token_count(&stream) == 7)

    token_stream_discard_a_token(&stream)
    token_stream_discard_a_token(&stream)
    token_stream_discard_a_token(&stream)

    assert(token_stream_remaining_token_count(&stream) == 4)

    token_stream_mark(&stream)  # 3

    token_stream_discard_a_token(&stream)

    assert(token_stream_remaining_token_count(&stream) == 3)

    token_stream_restore_a_mark(&stream)  # Back to 3

    assert(token_stream_remaining_token_count(&stream) == 4)

    token_stream_discard_a_token(&stream)
    token_stream_discard_a_token(&stream)
    token_stream_discard_a_token(&stream)

    assert(token_stream_remaining_token_count(&stream) == 1)

    token_stream_mark(&stream)  # 6

    token_stream_discard_a_mark(&stream)

    assert(token_stream_remaining_token_count(&stream) == 1)

    token_stream_restore_a_mark(&stream)  # Back to 0

    assert(token_stream_remaining_token_count(&stream) == 7)

test "TestCSSTokenStream marks":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_token_stream_marks()

# mark: ComponentValue streams and transactions =================================================

## `rgb(1,  [x]/**/{a:b})` with the names and source texts of the donor probe: a Function
## holding a number, a comma, whitespace, a [] block, a comment-whitespace and a {} block.
func sample_function_value() -> ComponentValue:
    var inner_square = ak.Vector[ComponentValue]()
    ak.vector_append_t(&inner_square, component_value_from_token(token_create_ident(ak.fly_string_from_str("x"), ak.string_from_str("x"))))
    let square = SimpleBlock(token = token_create(TokenType.open_square, ak.string_from_str("[")), value = inner_square, end_token = token_create(TokenType.close_square, ak.string_from_str("]")))
    var inner_curly = ak.Vector[ComponentValue]()
    ak.vector_append_t(&inner_curly, component_value_from_token(token_create_ident(ak.fly_string_from_str("a"), ak.string_from_str("a"))))
    ak.vector_append_t(&inner_curly, component_value_from_token(token_create(TokenType.colon, ak.string_from_str(":"))))
    ak.vector_append_t(&inner_curly, component_value_from_token(token_create_ident(ak.fly_string_from_str("b"), ak.string_from_str("b"))))
    let curly = SimpleBlock(token = token_create(TokenType.open_curly, ak.string_from_str("{")), value = inner_curly, end_token = token_create(TokenType.close_curly, ak.string_from_str("}")))
    var arguments = ak.Vector[ComponentValue]()
    ak.vector_append_t(&arguments, component_value_from_token(token_create_number(number_create(NumberType.integer, 1.0), ak.string_from_str("1"))))
    ak.vector_append_t(&arguments, component_value_from_token(token_create(TokenType.comma, ak.string_from_str(","))))
    ak.vector_append_t(&arguments, component_value_from_token(token_create_whitespace(ak.string_from_str("  "))))
    ak.vector_append_t(&arguments, component_value_from_simple_block(square))
    ak.vector_append_t(&arguments, component_value_from_token(token_create_whitespace(ak.string_from_str("/**/"))))
    ak.vector_append_t(&arguments, component_value_from_simple_block(curly))
    let function = Function(name = ak.fly_string_from_str("rgb"), value = arguments, name_token = token_create_function(ak.fly_string_from_str("rgb"), ak.string_from_str("RGB(")), end_token = token_create(TokenType.close_paren, ak.string_from_str(")")))
    return component_value_from_function(function)

## Asserts `string` holds `text`.
func expect_text(string: ak.String, text: str):
    let actual = ak.string_view_text(ak.string_bytes_as_string_view(&string))
    if actual != text:
        print(f"expected: {text}")
        print(f"actual:   {actual}")
        assert(false, "string mismatch")

noinline func test_component_values():
    let value = sample_function_value()
    expect_text(component_value_to_string(&value), "rgb(1, [x] {a:b})")
    expect_text(component_value_to_debug_string(&value), "Function: rgb(1, [x] {a:b})")
    expect_text(component_value_original_source_text(&value), "RGB(1,  [x]/**/{a:b})")
    assert(component_value_is_function_string_view(&value, ak.sv("RGB")))
    assert(not component_value_is_function_string_view(&value, ak.sv("hsl")))
    assert(not component_value_contains_guaranteed_invalid_value(&value))
    assert(not component_value_contains_attr_tainted_value(&value))
    let function = component_value_function(&value)
    let arguments = ak.vector_span_const(&function.value)
    expect_text(component_value_to_debug_string(&arguments[3]), "Block: [x]")
    expect_text(component_value_to_debug_string(&arguments[0]), "Token: Number(value=1, number_type=Integer, source=\"1\", start=0:0, end=0:0)")
    let square = component_value_block(&arguments[3])
    assert(component_value_is_block(&arguments[3]) and simple_block_is_square(&square))
    assert(component_value_is(&arguments[1], TokenType.comma))

    let invalid = component_value_from_guaranteed_invalid_value(GuaranteedInvalidValue())
    expect_text(component_value_to_debug_string(&invalid), "Guaranteed-invalid value")
    expect_text(component_value_to_string(&invalid), "")
    assert(component_value_is_guaranteed_invalid(&invalid))

    let delim = component_value_from_token(token_create_delim(0x2f, ak.string_new()))
    assert(component_value_is_delim(&delim, 0x2f))
    assert(not component_value_is_delim(&delim, 0x2a))
    let auto_ident = component_value_from_token(ident_token("Auto"))
    assert(component_value_is_ident(&auto_ident, ak.sv("auto")))

    # A guaranteed-invalid value or an attr-tainted value anywhere inside marks the whole value.
    var nested = ak.Vector[ComponentValue]()
    ak.vector_append_t(&nested, delim)
    ak.vector_append_t(&nested, invalid)
    let block = SimpleBlock(token = token_create(TokenType.open_paren, ak.string_new()), value = nested, end_token = token_new())
    let block_value = component_value_from_simple_block(block)
    assert(component_value_contains_guaranteed_invalid_value(&block_value))
    var tainted = component_value_from_token(ident_token("t"))
    component_value_set_attr_tainted(&tainted)
    var tainted_list = ak.Vector[ComponentValue]()
    ak.vector_append_t(&tainted_list, tainted)
    let tainted_function = component_value_from_function(Function(name = ak.fly_string_from_str("f"), value = tainted_list, name_token = token_new(), end_token = token_new()))
    assert(component_value_contains_attr_tainted_value(&tainted_function))
    assert(not component_value_contains_attr_tainted_value(&block_value))

    # operator==: the alternative, its value, and the attr-taint flag.
    let same = sample_function_value()
    assert(component_value_eq(&value, &same))
    assert(not component_value_eq(&value, &delim))
    let untainted = component_value_from_token(ident_token("t"))
    assert(not component_value_eq(&tainted, &untainted))
    assert(component_value_eq(&invalid, &invalid))

test "ComponentValue: strings, source text, lookups and equality":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_component_values()

noinline func test_component_value_stream():
    var list = ak.Vector[ComponentValue]()
    ak.vector_append_t(&list, component_value_from_token(token_create_whitespace(ak.string_from_str(" "))))
    ak.vector_append_t(&list, component_value_from_token(ident_token("a")))
    ak.vector_append_t(&list, sample_function_value())
    ak.vector_append_t(&list, component_value_from_token(token_create_whitespace(ak.string_from_str(" "))))
    var stream = token_stream_create(list)
    expect_text(token_stream_dump_string(&stream), " argb(1, [x] {a:b}) ")
    token_stream_discard_whitespace(&stream)
    let next = token_stream_next_token(&stream)
    expect_text(component_value_to_string(&next), "a")
    assert(token_stream_remaining_token_count(&stream) == 3)

    # A transaction that is not committed restores the index when it ends.
    if true:
        var transaction = token_stream_begin_transaction(&stream)
        defer state_transaction_destroy(&transaction)
        _ = token_stream_consume_a_token(&stream)
        _ = token_stream_consume_a_token(&stream)
        assert(token_stream_current_index(&stream) == 3)
    assert(token_stream_current_index(&stream) == 1)

    # Committing a child commits its parent.
    if true:
        var transaction = token_stream_begin_transaction(&stream)
        defer state_transaction_destroy(&transaction)
        _ = token_stream_consume_a_token(&stream)
        var child = state_transaction_create_child(&transaction)
        defer state_transaction_destroy(&child)
        _ = token_stream_consume_a_token(&stream)
        state_transaction_commit(&child)
    assert(token_stream_current_index(&stream) == 3)
    assert(token_stream_tokens_since(&stream, 0).length == 3)
    assert(token_stream_tokens_since(&stream, 4).length == 0)

    let last = token_stream_consume_a_token(&stream)
    expect_text(component_value_to_debug_string(&last), "Token: Whitespace(source=\" \", start=0:0, end=0:0)")
    assert(token_stream_is_empty(&stream))
    # Past the end, the stream answers its EOF: a ComponentValue holding an EOF token.
    let eof = token_stream_peek_token(&stream, 5)
    assert(component_value_is(&eof, TokenType.end_of_file))

    # of_single_token views its one token.
    let single_token = ident_token("only")
    var single = token_stream_of_single_token(&single_token)
    assert(token_stream_remaining_token_count(&single) == 1)
    let only = token_stream_consume_a_token(&single)
    assert(is_ident_named(&only, "only"))
    assert(token_stream_is_empty(&single))

test "TokenStream of ComponentValues, transactions and tokens_since":
    # Interned FlyStrings outlive the per-test allocator (docs/regions/r02.md), so the
    # test allocates from the process heap.
    test_component_value_stream()