dymokomi/luce-js
Source of release 0.6.1
0.6.1 / src / engine / tokenizer_next.lucb
#==============================================================================================
#
# tokenizer_next - next_token: the JavaScript tokenizer
#
# DESCRIPTION:
# Port of quickjs.c next_token (lines 22829-23270). QuickJS's labels become a
# `while true:` loop around the character switch: `goto redo` is `continue`, and
# the shared tails (identifiers, numbers, line comments) are the helpers below.
# On failure the token is TOK_ERROR.
#
#==============================================================================================
# mark: Shared tails ============================================================================
# Mark the current token as the error token after a failure.
func set_token_error(s: ParseState*):
s.token.val = TokenType.error_
# An identifier token whose first character `c` was read; `*pp` is after it (has_ident).
func next_token_ident(s: ParseState*, pp: const u8**, c: i32, ident_has_escape: bool) -> !:
var has_escape = ident_has_escape
let atom = try parse_ident(s, pp, &has_escape, c, false)
s.token.u.ident.atom = atom
s.token.u.ident.has_escape = has_escape
s.token.u.ident.is_reserved = false
s.token.val = TokenType.ident
update_token_ident(s)
# A number token at `*pp`, which moves past it (parse_number).
func next_token_number(s: ParseState*, pp: const u8**) -> !:
var p = *pp
let flags = (atod_accept_bin_oct | atod_accept_legacy_octal | atod_accept_underscores | atod_accept_suffix)
let ret = try js_atof(s.ctx, p, &p, 0, flags)
# reject `10instanceof Number`
var p1 = p
if (js_value_is_nan(ret) or unicode.lre_js_is_ident_next((u32)cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p1))):
js_free_value(s.ctx, ret)
try js_parse_error(s, "invalid number literal")
s.token.val = TokenType.number
s.token.u.num.val = ret
*pp = p
# The position of the line terminator (or the end) that ends the comment at `p0`
# (skip_line_comment).
func skip_line_comment(s: ParseState*, p0: const u8*) -> const u8*:
var p = p0
while true:
if *p == 0 and p >= s.buf_end:
break
if *p == '\r' or *p == '\n':
break
if *p >= 0x80:
let c = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
# LS or PS are considered as line terminator
if c == (i32)cp_ls or c == (i32)cp_ps:
break
elif c == -1:
# skip invalid UTF-8
p += 1
else:
p += 1
return p
# mark: next_token ==============================================================================
# Read the next token of the source into s.token.
func next_token(s: ParseState*) -> !:
if js_check_stack_overflow(s.ctx.rt, 0):
try js_parse_error(s, "stack overflow")
free_token(s, &s.token)
errdefer set_token_error(s)
var p = s.buf_ptr
s.last_ptr = p
s.got_lf = false
# redo:
while true:
s.token.ptr = p
let c = *p
match c:
0:
if p >= s.buf_end:
s.token.val = TokenType.eof
else:
s.token.val = (TokenType)(i32)c
p += 1
'`':
try js_parse_template_part(s, p + 1)
p = s.buf_ptr
'\'', '"':
try js_parse_string(s, (i32)c, true, p + 1, &s.token, &p)
'\r', '\n':
# accept DOS and MAC newline sequences
if c == '\r' and p[1] == '\n':
p += 1
p += 1
# line_terminator:
s.got_lf = true
continue
0x0c, 0x0b, ' ', '\t':
p += 1
continue
'/':
if p[1] == '*':
# comment
p += 2
while true:
if *p == 0 and p >= s.buf_end:
try js_parse_error(s, "unexpected end of comment")
if p[0] == '*' and p[1] == '/':
p += 2
break
if *p == '\n' or *p == '\r':
# considered as LF for ASI
s.got_lf = true
p += 1
elif *p >= 0x80:
let c1 = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
if c1 == (i32)cp_ls or c1 == (i32)cp_ps:
# considered as LF for ASI
s.got_lf = true
elif c1 == -1:
# skip invalid UTF-8
p += 1
else:
p += 1
continue
elif p[1] == '/':
# line comment
p = skip_line_comment(s, p + 2)
continue
elif p[1] == '=':
p += 2
s.token.val = TokenType.div_assign
else:
p += 1
s.token.val = (TokenType)(i32)c
'\\':
var is_ident = false
if p[1] == 'u':
var p1 = p + 1
let c1 = lre_parse_escape(&p1, 1)
if c1 >= 0 and unicode.lre_js_is_ident_first((u32)c1):
p = p1
is_ident = true
try next_token_ident(s, &p, c1, true)
# else XXX: syntax error?
if not is_ident:
s.token.val = (TokenType)(i32)c
p += 1
'a'..='z', 'A'..='Z', '_', '$':
# identifier
p += 1
try next_token_ident(s, &p, (i32)c, false)
'#':
# private name
p += 1
var p1 = p
var c1 = (i32)*p1
p1 += 1
if c1 == (i32)'\\' and *p1 == 'u':
c1 = lre_parse_escape(&p1, 1)
elif c1 >= 128:
c1 = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p1)
if not unicode.lre_js_is_ident_first((u32)c1):
try js_parse_error(s, "invalid first character of private name")
p = p1
# not used
var ident_has_escape = false
let atom = try parse_ident(s, &p, &ident_has_escape, c1, true)
s.token.u.ident.atom = atom
s.token.val = TokenType.private_name
'.':
if p[1] == '.' and p[2] == '.':
p += 3
s.token.val = TokenType.ellipsis
elif p[1] >= '0' and p[1] <= '9':
try next_token_number(s, &p)
else:
s.token.val = (TokenType)(i32)c
p += 1
'0':
# in strict mode, octal literals are not accepted
if is_digit((i32)p[1]) and ((i32)s.fd().js_mode & js_mode_strict) != 0:
try js_parse_error(s, "octal literals are deprecated in strict mode")
try next_token_number(s, &p)
'1'..='9':
# number
try next_token_number(s, &p)
'*':
if p[1] == '=':
p += 2
s.token.val = TokenType.mul_assign
elif p[1] == '*':
if p[2] == '=':
p += 3
s.token.val = TokenType.pow_assign
else:
p += 2
s.token.val = TokenType.pow
else:
s.token.val = (TokenType)(i32)c
p += 1
'%':
if p[1] == '=':
p += 2
s.token.val = TokenType.mod_assign
else:
s.token.val = (TokenType)(i32)c
p += 1
'+':
if p[1] == '=':
p += 2
s.token.val = TokenType.plus_assign
elif p[1] == '+':
p += 2
s.token.val = TokenType.inc
else:
s.token.val = (TokenType)(i32)c
p += 1
'-':
if p[1] == '=':
p += 2
s.token.val = TokenType.minus_assign
elif p[1] == '-':
if (s.allow_html_comments and p[2] == '>' and (s.got_lf or s.last_ptr == s.buf_start)):
# Annex B: `-->` at beginning of line is an html comment end.
# It extends to the end of the line.
p = skip_line_comment(s, p)
continue
p += 2
s.token.val = TokenType.dec
else:
s.token.val = (TokenType)(i32)c
p += 1
'<':
if p[1] == '=':
p += 2
s.token.val = TokenType.lte
elif p[1] == '<':
if p[2] == '=':
p += 3
s.token.val = TokenType.shl_assign
else:
p += 2
s.token.val = TokenType.shl
elif s.allow_html_comments and p[1] == '!' and p[2] == '-' and p[3] == '-':
# Annex B: handle `<!--` single line html comments
p = skip_line_comment(s, p)
continue
else:
s.token.val = (TokenType)(i32)c
p += 1
'>':
if p[1] == '=':
p += 2
s.token.val = TokenType.gte
elif p[1] == '>':
if p[2] == '>':
if p[3] == '=':
p += 4
s.token.val = TokenType.shr_assign
else:
p += 3
s.token.val = TokenType.shr
elif p[2] == '=':
p += 3
s.token.val = TokenType.sar_assign
else:
p += 2
s.token.val = TokenType.sar
else:
s.token.val = (TokenType)(i32)c
p += 1
'=':
if p[1] == '=':
if p[2] == '=':
p += 3
s.token.val = TokenType.strict_eq
else:
p += 2
s.token.val = TokenType.eq
elif p[1] == '>':
p += 2
s.token.val = TokenType.arrow
else:
s.token.val = (TokenType)(i32)c
p += 1
'!':
if p[1] == '=':
if p[2] == '=':
p += 3
s.token.val = TokenType.strict_neq
else:
p += 2
s.token.val = TokenType.neq
else:
s.token.val = (TokenType)(i32)c
p += 1
'&':
if p[1] == '=':
p += 2
s.token.val = TokenType.and_assign
elif p[1] == '&':
if p[2] == '=':
p += 3
s.token.val = TokenType.land_assign
else:
p += 2
s.token.val = TokenType.land
else:
s.token.val = (TokenType)(i32)c
p += 1
'^':
if p[1] == '=':
p += 2
s.token.val = TokenType.xor_assign
else:
s.token.val = (TokenType)(i32)c
p += 1
'|':
if p[1] == '=':
p += 2
s.token.val = TokenType.or_assign
elif p[1] == '|':
if p[2] == '=':
p += 3
s.token.val = TokenType.lor_assign
else:
p += 2
s.token.val = TokenType.lor
else:
s.token.val = (TokenType)(i32)c
p += 1
'?':
if p[1] == '?':
if p[2] == '=':
p += 3
s.token.val = TokenType.double_question_mark_assign
else:
p += 2
s.token.val = TokenType.double_question_mark
elif p[1] == '.' and not (p[2] >= '0' and p[2] <= '9'):
p += 2
s.token.val = TokenType.question_mark_dot
else:
s.token.val = (TokenType)(i32)c
p += 1
_:
if c >= 128:
# unicode value
let c1 = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
if c1 == (i32)cp_ps or c1 == (i32)cp_ls:
# XXX: should avoid incrementing line_number, but needed to
# handle HTML comments
# line_terminator:
s.got_lf = true
continue
elif unicode.lre_is_space((u32)c1):
continue
elif unicode.lre_js_is_ident_first((u32)c1):
try next_token_ident(s, &p, c1, false)
else:
try js_parse_error(s, "unexpected character")
else:
# def_token:
s.token.val = (TokenType)(i32)c
p += 1
break
s.buf_ptr = p
# dump_token(s, &s.token)