dymokomi/luce-browser-css
Source of release 0.5.1
0.5.1 / src / css_syntax / token_stream.lucb
#==============================================================================================
#
# token_stream - CSS::Parser::TokenStream<T>: the CSS token stream over Tokens or
# ComponentValues
#
# DESCRIPTION:
# Port of LibWeb/CSS/Parser/TokenStream.h (lines 1-212). The template's T is Token or
# ComponentValue; its calls on a T go through the TokenStreamToken interface
# (types_token_stream). The C++ returns `T const&` into the tokens (or into m_eof); the
# port returns the T by value, which shares the token's strings and vectors.
#
# StateTransaction's destructor is state_transaction_destroy, called where the C++
# transaction goes out of scope:
#
# var transaction = token_stream_begin_transaction(tokens)
# defer state_transaction_destroy(&transaction)
#
#==============================================================================================
# mark: StateTransaction ========================================================================
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:24 Web::CSS::Parser::TokenStream::StateTransaction::StateTransaction.
pub func state_transaction_construct[T](this: StateTransaction*, token_stream: TokenStream[T]*):
this.m_parent = none
this.m_token_stream_index = &token_stream.m_index
this.m_saved_index = token_stream.m_index
this.m_commit = false
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:30 Web::CSS::Parser::TokenStream::StateTransaction::~StateTransaction.
pub func state_transaction_destroy(this: StateTransaction*):
if not this.m_commit:
*this.m_token_stream_index = this.m_saved_index
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:36 Web::CSS::Parser::TokenStream::StateTransaction::create_child.
## The child refers to `this`, which must stay where it is while the child lives (as in C++).
pub func state_transaction_create_child(this: StateTransaction*) -> StateTransaction:
var child = StateTransaction(m_token_stream_index = this.m_token_stream_index)
state_transaction_construct_state_transaction(&child, this)
return child
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:38 Web::CSS::Parser::TokenStream::StateTransaction::commit.
pub func state_transaction_commit(this: StateTransaction*):
this.m_commit = true
if let parent = this.m_parent:
state_transaction_commit(parent)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:46 Web::CSS::Parser::TokenStream::StateTransaction::StateTransaction
## (the private constructor from a parent transaction).
func state_transaction_construct_state_transaction(this: StateTransaction*, parent: StateTransaction*):
this.m_parent = parent
this.m_token_stream_index = parent.m_token_stream_index
this.m_saved_index = *parent.m_token_stream_index
this.m_commit = false
# mark: TokenStream =============================================================================
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:58 Web::CSS::Parser::TokenStream::TokenStream<T>.
## The stream views `tokens`, which must outlive it (as in C++).
pub func token_stream_construct_span_t[T: TokenStreamToken[T]](this: TokenStream[T]*, tokens: const T[]):
this.m_tokens = tokens
this.m_index = 0
this.m_marked_indexes = ak.Vector[usize]()
this.m_eof = token_stream_make_eof(this)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:64 Web::CSS::Parser::TokenStream::TokenStream<T>.
## The stream views the vector's elements, which must outlive it and stay in place (as in C++).
pub func token_stream_construct_vector[T: TokenStreamToken[T]](this: TokenStream[T]*, tokens: ak.Vector[T]):
this.m_tokens = ak.vector_span_const(&tokens)
this.m_index = 0
this.m_marked_indexes = ak.Vector[usize]()
this.m_eof = token_stream_make_eof(this)
## `TokenStream stream { tokens }` over a span: the value form of token_stream_construct_span_t.
pub func token_stream_create_span[T: TokenStreamToken[T]](tokens: const T[]) -> TokenStream[T]:
# The constructor fills every field; make_eof() does not read the stream.
var stream: TokenStream[T] = ---
token_stream_construct_span_t(&stream, tokens)
return stream
## `TokenStream stream { tokens }` over a vector: the value form of token_stream_construct_vector.
pub func token_stream_create[T: TokenStreamToken[T]](tokens: ak.Vector[T]) -> TokenStream[T]:
# The constructor fills every field; make_eof() does not read the stream.
var stream: TokenStream[T] = ---
token_stream_construct_vector(&stream, tokens)
return stream
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:70 Web::CSS::Parser::TokenStream::of_single_token.
## The stream views `token`, which must outlive it (C++ takes it by reference).
pub func token_stream_of_single_token[T: TokenStreamToken[T]](token: const T*) -> TokenStream[T]:
return token_stream_create_span(T[]((T*)token, 1))
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:79 Web::CSS::Parser::TokenStream::next_token.
# https://drafts.csswg.org/css-syntax/#token-stream-next-token
pub func token_stream_next_token[T](this: const TokenStream[T]*) -> T:
# The item of tokens at index.
# If that index would be out-of-bounds past the end of the list, it’s instead an <eof-token>.
if this.m_index < this.m_tokens.length:
return this.m_tokens[this.m_index]
return this.m_eof
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:89 Web::CSS::Parser::TokenStream::peek_token.
## NB: If offset is 0, this is the same as next_token().
pub func token_stream_peek_token[T](this: TokenStream[T]*, offset: usize) -> T:
if this.m_index + offset < this.m_tokens.length:
return this.m_tokens[this.m_index + offset]
return this.m_eof
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:97 Web::CSS::Parser::TokenStream::is_empty.
# https://drafts.csswg.org/css-syntax/#token-stream-empty
pub func token_stream_is_empty[T: TokenStreamToken[T]](this: const TokenStream[T]*) -> bool:
# A token stream is empty if the next token is an <eof-token>.
let next = token_stream_next_token(this)
return next.token_stream_is(TokenType.end_of_file)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:104 Web::CSS::Parser::TokenStream::consume_a_token.
# https://drafts.csswg.org/css-syntax/#token-stream-consume-a-token
pub func token_stream_consume_a_token[T](this: TokenStream[T]*) -> T:
# Let token be the next token. Increment index, then return token.
let token = token_stream_next_token(this)
this.m_index += 1
return token
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:113 Web::CSS::Parser::TokenStream::discard_a_token.
# https://drafts.csswg.org/css-syntax/#token-stream-discard-a-token
pub func token_stream_discard_a_token[T: TokenStreamToken[T]](this: TokenStream[T]*):
# If the token stream is not empty, increment index.
if not token_stream_is_empty(this):
this.m_index += 1
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:121 Web::CSS::Parser::TokenStream::mark.
# https://drafts.csswg.org/css-syntax/#token-stream-mark
pub func token_stream_mark[T](this: TokenStream[T]*):
# Append index to marked indexes.
ak.vector_append_t(&this.m_marked_indexes, this.m_index)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:128 Web::CSS::Parser::TokenStream::restore_a_mark.
# https://drafts.csswg.org/css-syntax/#token-stream-restore-a-mark
pub func token_stream_restore_a_mark[T](this: TokenStream[T]*):
# Pop from marked indexes, and set index to the popped value.
this.m_index = ak.vector_take_last(&this.m_marked_indexes)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:135 Web::CSS::Parser::TokenStream::discard_a_mark.
# https://drafts.csswg.org/css-syntax/#token-stream-discard-a-mark
pub func token_stream_discard_a_mark[T](this: TokenStream[T]*):
# Pop from marked indexes, and do nothing with the popped value.
_ = ak.vector_take_last(&this.m_marked_indexes)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:142 Web::CSS::Parser::TokenStream::discard_whitespace.
# https://drafts.csswg.org/css-syntax/#token-stream-discard-whitespace
pub func token_stream_discard_whitespace[T: TokenStreamToken[T]](this: TokenStream[T]*):
# While the next token is a <whitespace-token>, discard a token.
while token_stream_next_token(this).token_stream_is(TokenType.whitespace):
token_stream_discard_a_token(this)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:149 Web::CSS::Parser::TokenStream::has_next_token.
pub func token_stream_has_next_token[T: TokenStreamToken[T]](this: TokenStream[T]*) -> bool:
return not token_stream_is_empty(this)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:154 Web::CSS::Parser::TokenStream::begin_transaction.
pub func token_stream_begin_transaction[T](this: TokenStream[T]*) -> StateTransaction:
var transaction = StateTransaction(m_token_stream_index = &this.m_index)
state_transaction_construct(&transaction, this)
return transaction
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:156 Web::CSS::Parser::TokenStream::remaining_token_count.
pub func token_stream_remaining_token_count[T](this: const TokenStream[T]*) -> usize:
if this.m_tokens.length > this.m_index:
return this.m_tokens.length - this.m_index
return 0
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:163 Web::CSS::Parser::TokenStream::current_index.
pub func token_stream_current_index[T](this: const TokenStream[T]*) -> usize:
return this.m_index
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:165 Web::CSS::Parser::TokenStream::tokens_since.
pub func token_stream_tokens_since[T](this: const TokenStream[T]*, start: usize) -> const T[]:
if start > this.m_index:
var empty: const T[]
return empty
return this.m_tokens[start..<this.m_index]
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:172 Web::CSS::Parser::TokenStream::dump_all_tokens.
pub func token_stream_dump_all_tokens[T: TokenStreamToken[T]](this: TokenStream[T]*):
ak.dbgln("Dumping all tokens:")
for i in 0..<this.m_tokens.length:
let token = &this.m_tokens[i]
let debug_string = token.token_stream_to_debug_string()
let text = ak.string_view_text(ak.string_bytes_as_string_view(&debug_string))
if i == this.m_index:
ak.dbgln(f"-> {text}")
else:
ak.dbgln(f" {text}")
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:184 Web::CSS::Parser::TokenStream::dump_string.
pub func token_stream_dump_string[T: TokenStreamToken[T]](this: TokenStream[T]*) -> ak.String:
# MUST(String::join(""sv, m_tokens)): each token through its Formatter (to_string).
var builder = ak.string_builder_new()
for i in 0..<this.m_tokens.length:
let string = this.m_tokens[i].token_stream_to_string()
string_builder_append_string(&builder, &string)
return ak.string_builder_to_string_without_validation(&builder)
## Ported from Libraries/LibWeb/CSS/Parser/TokenStream.h:199 Web::CSS::Parser::TokenStream::make_eof.
func token_stream_make_eof[T: TokenStreamToken[T]](this: TokenStream[T]*) -> T:
# if constexpr (IsSame<T, Token>) Tokenizer::create_eof_token(), and for ComponentValue
# ComponentValue(Tokenizer::create_eof_token()): the T's token_stream_make_eof, which does not
# read its receiver (m_eof may not be constructed yet).
return this.m_eof.token_stream_make_eof()