dymokomi/luce-js
Source of release 0.6.0
0.6.0 / src / engine / tokenizer.lucb
#==============================================================================================
#
# tokenizer - Tokens, source positions, parse errors and the literal scanners
#
# DESCRIPTION:
# Port of quickjs.c "JS parser" lines 22188-22826: releasing tokens,
# line and column numbers, the js_parse_error family, js_parse_expect, template,
# string and regular expression literals, and identifiers (with the keyword
# conversion). next_token is in tokenizer_next.lucb.
#
# A C function that answers -1 on a syntax error is fallible here: the error is a
# SyntaxError thrown in the context. The literal scanners run their loop in a
# helper that the caller wraps, so the StringBuffer is freed on every failure and
# never after string_buffer_end consumed it.
#
#==============================================================================================
# mark: Tokens ==================================================================================
# Whether `tok` is a keyword token (TOK_FIRST_KEYWORD to TOK_LAST_KEYWORD).
func tok_is_keyword(tok: TokenType) -> bool:
return (i32)tok >= (i32)tok_first_keyword and (i32)tok <= (i32)tok_last_keyword
# The character a one-character token is, as printf's "%c" prints it: the low byte,
# read as Latin-1.
func tok_char(tok: TokenType) -> char:
return char((u32)(u8)(i32)tok)
func free_token(s: ParseState*, token: Token*):
match token.val:
.number:
js_free_value(s.ctx, token.u.num.val)
.string, .template:
js_free_value(s.ctx, token.u.str_.str)
.regexp:
js_free_value(s.ctx, token.u.regexp.body)
js_free_value(s.ctx, token.u.regexp.flags)
.ident, .private_name:
js_free_atom(s.ctx, token.u.ident.atom)
_:
if tok_is_keyword(token.val):
js_free_atom(s.ctx, token.u.ident.atom)
# mark: Source positions ========================================================================
# return the zero based line and column number in the source.
# Note: we no longer support '\r' as line terminator
func get_line_col(pcol_num: i32*, buf: const u8*, len: usize) -> i32:
var line_num: i32 = 0
var col_num: i32 = 0
for i in 0..<len:
let c = buf[i]
if c == '\n':
line_num += 1
col_num = 0
elif c < 0x80 or c >= 0xc0:
col_num += 1
*pcol_num = col_num
return line_num
# The zero based line and column of `ptr`, computed from the last position the cache
# holds, which moves to `ptr`.
func get_line_col_cached(s: GetLineColCache*, pcol_num: i32*, ptr: const u8*) -> i32:
var col_num: i32 = 0
if ptr >= s.ptr:
let line_num = get_line_col(&col_num, s.ptr, (usize)(ptr - s.ptr))
if line_num == 0:
s.col_num += col_num
else:
s.line_num += line_num
s.col_num = col_num
else:
let line_num = get_line_col(&col_num, ptr, (usize)(s.ptr - ptr))
if line_num == 0:
s.col_num -= col_num
else:
s.line_num -= line_num
# find the absolute column position
col_num = 0
var p = ptr - 1
while p >= s.buf_start:
if *p == '\n':
break
elif *p < 0x80 or *p >= 0xc0:
col_num += 1
p -= 1
s.col_num = col_num
s.ptr = ptr
*pcol_num = s.col_num
return s.line_num
# mark: Parse errors ============================================================================
# 'ptr' is the position of the error in the source. Throw a SyntaxError whose backtrace
# names that position.
func js_parse_error_v(s: ParseState*, ptr: const u8*, message: fmt) -> never!:
let ctx = s.ctx
var col_num: i32 = 0
let line_num = get_line_col(&col_num, s.buf_start, (usize)(ptr - s.buf_start))
js_throw_error2(ctx, ErrorKind.syntax_error, message, false) catch failure:
build_backtrace(ctx, ctx.rt.current_exception, s.filename, line_num + 1, col_num + 1, 0)
error(failure.code, failure.message)
# Throw a SyntaxError at `ptr`.
func js_parse_error_pos(s: ParseState*, ptr: const u8*, message: fmt) -> never!:
try js_parse_error_v(s, ptr, message)
# Throw a SyntaxError at the current token.
func js_parse_error(s: ParseState*, message: fmt) -> never!:
try js_parse_error_v(s, s.token.ptr, message)
# Read the next token after checking that the current one is `tok`.
func js_parse_expect(s: ParseState*, tok: TokenType) -> !:
if s.token.val != tok:
# XXX: dump token correctly in all cases
try js_parse_error(s, f"expecting '{tok_char(tok)}'")
try next_token(s)
# Read the next token after a ';', or accept its automatic insertion.
func js_parse_expect_semi(s: ParseState*) -> !:
if s.token.val != TokenType.semicolon:
# automatic insertion of ';'
if s.token.val == TokenType.eof or s.token.val == TokenType.rbrace or s.got_lf:
return
try js_parse_error(s, "expecting ';'")
try next_token(s)
func js_parse_error_reserved_identifier(s: ParseState*) -> never!:
var buf1: u8[atom_get_str_buf_size] = ---
try js_parse_error(s, f"'{js_atom_get_str(s.ctx, buf1, s.token.u.ident.atom)}' is a reserved identifier")
# mark: Template and string literals ============================================================
# The characters of a template part into `b`, from its first byte `p0`: answers the
# position after the part and its last character ('`' at the end, '$' before a
# substitution).
func js_parse_template_chars(s: ParseState*, b: StringBuffer*, p0: const u8*) -> (const u8*, u32)!:
var p = p0
var c: u32 = 0
while true:
if p >= s.buf_end:
try js_parse_error(s, "unexpected end of string")
c = (u32)*p
p += 1
if c == (u32)'`':
# template end part
break
if c == (u32)'$' and *p == '{':
# template start or middle part
p += 1
break
if c == (u32)'\\':
try string_buffer_putc8(b, c)
if p >= s.buf_end:
try js_parse_error(s, "unexpected end of string")
c = (u32)*p
p += 1
# newline sequences are normalized as single '\n' bytes
if c == (u32)'\r':
if *p == '\n':
p += 1
c = (u32)'\n'
if c >= 0x80:
var p_next = p
c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
try js_parse_error_pos(s, p - 1, "invalid UTF-8 sequence")
p = p_next
try string_buffer_putc(b, c)
return (p, c)
# Scan a template part; 'p' points to the first byte of the template part.
func js_parse_template_part(s: ParseState*, p: const u8*) -> !:
var b: StringBuffer = ---
string_buffer_init(s.ctx, &b, 32) catch failure:
string_buffer_free(&b)
error(failure.code, failure.message)
let (p_end, c) = js_parse_template_chars(s, &b, p) catch failure:
string_buffer_free(&b)
error(failure.code, failure.message)
let text = try string_buffer_end(&b)
s.token.val = TokenType.template
s.token.u.str_.sep = (i32)c
s.token.u.str_.str = text
s.buf_ptr = p_end
# A malformed string literal: a SyntaxError at `ptr` when `do_throw`, else a failure
# with no exception.
func js_parse_string_fail(s: ParseState*, do_throw: bool, ptr: const u8*, message: fmt) -> never!:
if do_throw:
try js_parse_error_pos(s, ptr, message)
error(exception, "")
# The characters of a string literal into `b`, from the byte after the opening quote:
# answers the position after the literal and its last character.
func js_parse_string_chars(s: ParseState*, b: StringBuffer*, sep: i32, do_throw: bool,
p0: const u8*) -> (const u8*, u32)!:
var p = p0
var c: u32 = 0
while true:
if p >= s.buf_end:
try js_parse_string_fail(s, do_throw, s.token.ptr, "unexpected end of string")
c = (u32)*p
if c < 0x20:
if sep == (i32)'`':
if c == (u32)'\r':
if p[1] == '\n':
p += 1
c = (u32)'\n'
# do not update s->line_num
elif c == (u32)'\n' or c == (u32)'\r':
try js_parse_string_fail(s, do_throw, s.token.ptr, "unexpected end of string")
p += 1
if c == (u32)sep:
break
if c == (u32)'$' and *p == '{' and sep == (i32)'`':
# template start or middle part
p += 1
break
if c == (u32)'\\':
let p_escape = p - 1
c = (u32)*p
# XXX: need a specific JSON case to avoid accepting invalid escapes
match *p:
0:
if p >= s.buf_end:
try js_parse_string_fail(s, do_throw, s.token.ptr, "unexpected end of string")
p += 1
'\'', '"', '\\':
p += 1
'\r', '\n':
# accept DOS and MAC newline sequences
if c == (u32)'\r' and p[1] == '\n':
p += 1
# ignore escaped newline sequence
p += 1
continue
_:
var parse_escape = false
if c >= (u32)'0' and c <= (u32)'9':
if ((i32)s.fd().js_mode & js_mode_strict) == 0 and sep != (i32)'`':
parse_escape = true
elif c == (u32)'0' and not (p[1] >= '0' and p[1] <= '9'):
p += 1
c = 0
elif c >= (u32)'8' or sep == (i32)'`':
# Note: according to ES2021, \8 and \9 are not accepted in
# strict mode or in templates.
try js_parse_string_fail(s, do_throw, p_escape, "malformed escape sequence in string literal")
else:
try js_parse_string_fail(s, do_throw, p_escape, "octal escape sequences are not allowed in strict mode")
elif c >= 0x80:
var p_next = p
c = (u32)cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
try js_parse_string_fail(s, do_throw, s.token.ptr, "invalid UTF-8 sequence")
p = p_next
# LS or PS are skipped
if c == cp_ls or c == cp_ps:
continue
else:
parse_escape = true
if parse_escape:
let ret = lre_parse_escape(&p, 1)
if ret == -1:
try js_parse_string_fail(s, do_throw, p_escape, "malformed escape sequence in string literal")
elif ret < 0:
# ignore the '\' (could output a warning)
p += 1
else:
c = (u32)ret
elif c >= 0x80:
var p_next = p
c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
try js_parse_string_fail(s, do_throw, s.token.ptr, "invalid UTF-8 sequence")
p = p_next
try string_buffer_putc(b, c)
return (p, c)
# Scan a string literal delimited by `sep` from `p` (after the opening quote) into
# `token`, and store the position after it in `*pp`. When `do_throw` is false a
# malformed literal fails without an exception.
func js_parse_string(s: ParseState*, sep: i32, do_throw: bool, p: const u8*, token: Token*,
pp: const u8**) -> !:
var b: StringBuffer = ---
# string
string_buffer_init(s.ctx, &b, 32) catch failure:
string_buffer_free(&b)
error(failure.code, failure.message)
let (p_end, c) = js_parse_string_chars(s, &b, sep, do_throw, p) catch failure:
string_buffer_free(&b)
error(failure.code, failure.message)
let text = try string_buffer_end(&b)
token.val = TokenType.string
token.u.str_.sep = (i32)c
token.u.str_.str = text
*pp = p_end
func token_is_pseudo_keyword(s: ParseState*, atom: Atom) -> bool:
return (s.token.val == TokenType.ident and s.token.u.ident.atom == atom and not s.token.u.ident.has_escape)
# mark: Regular expression literals =============================================================
# The body of a regular expression literal into `b` and its flags into `b2`, from the
# byte after the opening '/': answers the position after the flags.
func js_parse_regexp_chars(s: ParseState*, b: StringBuffer*, b2: StringBuffer*, p0: const u8*) -> const u8*!:
var p = p0
var in_class = false
while true:
if p >= s.buf_end:
try js_parse_error(s, "unexpected end of regexp")
var c = (u32)*p
p += 1
if c == (u32)'\n' or c == (u32)'\r':
try js_parse_error_pos(s, p - 1, "unexpected line terminator in regexp")
elif c == (u32)'/':
if not in_class:
break
elif c == (u32)'[':
in_class = true
elif c == (u32)']':
# XXX: incorrect as the first character in a class
in_class = false
elif c == (u32)'\\':
try string_buffer_putc8(b, c)
c = (u32)*p
p += 1
if c == (u32)'\n' or c == (u32)'\r':
try js_parse_error_pos(s, p - 1, "unexpected line terminator in regexp")
elif c == 0 and p >= s.buf_end:
try js_parse_error(s, "unexpected end of regexp")
elif c >= 0x80:
var p_next = p
c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
try js_parse_error_pos(s, p - 1, "invalid UTF-8 sequence")
p = p_next
if c == cp_ls or c == cp_ps:
try js_parse_error_pos(s, p - 1, "unexpected line terminator in regexp")
elif c >= 0x80:
var p_next = p
c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
try js_parse_error_pos(s, p - 1, "invalid UTF-8 sequence")
# LS or PS are considered as line terminator
if c == cp_ls or c == cp_ps:
try js_parse_error_pos(s, p - 1, "unexpected line terminator in regexp")
p = p_next
try string_buffer_putc(b, c)
# flags
while true:
var p_next = p
var c = (u32)*p_next
p_next += 1
if c >= 0x80:
c = (u32)cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p_next)
if c > 0x10FFFF:
p += 1
try js_parse_error_pos(s, p - 1, "invalid UTF-8 sequence")
if not unicode.lre_js_is_ident_next(c):
break
try string_buffer_putc(b2, c)
p = p_next
return p
# Scan the regular expression literal at s.buf_ptr (its opening '/').
func js_parse_regexp(s: ParseState*) -> !:
var b: StringBuffer = ---
var b2: StringBuffer = ---
let p = s.buf_ptr + 1
try string_buffer_init(s.ctx, &b, 32)
string_buffer_init(s.ctx, &b2, 1) catch failure:
string_buffer_free(&b)
string_buffer_free(&b2)
error(failure.code, failure.message)
let p_end = js_parse_regexp_chars(s, &b, &b2, p) catch failure:
string_buffer_free(&b)
string_buffer_free(&b2)
error(failure.code, failure.message)
let body_str = string_buffer_end(&b) catch failure:
recover js_exception()
let flags_str = string_buffer_end(&b2) catch failure:
recover js_exception()
if js_is_exception(body_str) or js_is_exception(flags_str):
js_free_value(s.ctx, body_str)
js_free_value(s.ctx, flags_str)
error(exception, "")
s.token.val = TokenType.regexp
s.token.u.regexp.body = body_str
s.token.u.regexp.flags = flags_str
s.buf_ptr = p_end
# mark: Identifiers =============================================================================
# Grow the identifier buffer `*pbuf` of `*psize` bytes by half; the first growth copies
# it out of the caller's `static_buf`.
func ident_realloc(ctx: Context*, pbuf: u8**, psize: usize*, static_buf: u8*) -> !:
let buf = *pbuf
let size = *psize
var new_size: usize = 0
if size >= (~(usize)0 // 3) * 2:
new_size = ~(usize)0
else:
new_size = size + (size >> 1)
var new_buf = buf
if buf == static_buf:
new_buf = (u8*)(try js_malloc(ctx, new_size))
_ = cutils.memcpy(new_buf, buf, size)
else:
let p = try js_realloc(ctx, buf, new_size) else trap("ident_realloc: empty block")
new_buf = (u8*)p
*pbuf = new_buf
*psize = new_size
# convert a TOK_IDENT to a keyword when needed
func update_token_ident(s: ParseState*):
let fd = s.fd()
let atom = s.token.u.ident.atom
var parent_generator = false
var parent_async = false
if let parent = fd.parent:
parent_generator = (parent.func_kind & js_func_generator) != 0
parent_async = ((parent.func_kind & js_func_async) != 0 or parent.func_type == ParseFunctionKind.class_static_init)
let in_arrow_params = fd.func_type == ParseFunctionKind.arrow and not fd.in_function_body
if (atom <= atom_last_keyword or (atom <= atom_last_strict_keyword and ((i32)fd.js_mode & js_mode_strict) != 0) or (atom == atom_yield and ((fd.func_kind & js_func_generator) != 0 or (in_arrow_params and parent_generator))) or (atom == atom_await and (s.is_module or (fd.func_kind & js_func_async) != 0 or fd.func_type == ParseFunctionKind.class_static_init or (in_arrow_params and parent_async)))):
if s.token.u.ident.has_escape:
s.token.u.ident.is_reserved = true
s.token.val = TokenType.ident
else:
# The keywords atoms are pre allocated
s.token.val = (TokenType)((i32)atom - 1 + (i32)tok_first_keyword)
# if the current token is an identifier or keyword, reparse it according to the current
# function type
func reparse_ident_token(s: ParseState*):
if s.token.val == TokenType.ident or tok_is_keyword(s.token.val):
s.token.val = TokenType.ident
s.token.u.ident.is_reserved = false
update_token_ident(s)
# 'c' is the first character. The atom of the identifier that continues at `*pp`,
# which moves past it.
func parse_ident(s: ParseState*, pp: const u8**, pident_has_escape: bool*, c0: i32,
is_private: bool) -> Atom!:
var c = c0
var p = *pp
var ident_buf: u8[128] = ---
let static_buf: u8* = &ident_buf[0]
var buf = static_buf
var ident_size: usize = 128
var ident_pos: usize = 0
var failed = false
if is_private:
buf[ident_pos] = '#'
ident_pos += 1
while true:
var p1 = p
if c < 128:
buf[ident_pos] = (u8)c
ident_pos += 1
else:
ident_pos += (usize)cutils.unicode_to_utf8(buf + ident_pos, (u32)c)
c = (i32)*p1
p1 += 1
if c == (i32)'\\' and *p1 == 'u':
c = lre_parse_escape(&p1, 1)
*pident_has_escape = true
elif c >= 128:
c = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p1)
if not unicode.lre_js_is_ident_next((u32)c):
break
p = p1
if ident_pos >= ident_size - (usize)cutils.utf8_char_len_max:
ident_realloc(s.ctx, &buf, &ident_size, static_buf) catch failure:
failed = true
recover ()
if failed:
break
var atom = no_atom
if not failed:
atom = js_new_atom_len(s.ctx, buf, ident_pos) catch failure:
failed = true
recover no_atom
if buf != static_buf:
js_free(s.ctx, buf)
*pp = p
if failed:
error(exception, "")
return atom