Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-js

Source of release 0.6.0

0.6.0 / src / engine / tokenizer_json.lucb

#==============================================================================================
#
#   tokenizer_json - The JSON tokenizer, the look-ahead scanner and module detection
#
#   DESCRIPTION:
#       Port of quickjs.c lines 23270-23806: json_next_token and its identifier, string
#       and number scanners (with the ext_json superset), simple_next_token (the
#       look-ahead used for ':', '=>', 'let' and 'function'), peek_token, skip_shebang
#       and JS_DetectModule.
#
#==============================================================================================

# mark: JSON literals ===========================================================================

# 'c' is the first character. The atom of the identifier that continues at `*pp`,
# which moves past it.
# XXX: accept unicode identifiers as JSON5 ?
func json_parse_ident(s: ParseState*, pp: const u8**, c0: i32) -> Atom!:
    var c = c0
    var p = *pp
    var ident_buf: u8[128] = ---
    let static_buf: u8* = &ident_buf[0]
    var buf = static_buf
    var ident_size: usize = 128
    var ident_pos: usize = 0
    var failed = false
    while true:
        buf[ident_pos] = (u8)c
        ident_pos += 1
        c = (i32)*p
        if c >= 128 or not unicode.lre_is_id_continue_byte((u8)c):
            break
        p += 1
        if ident_pos >= ident_size - (usize)cutils.utf8_char_len_max:
            ident_realloc(s.ctx, &buf, &ident_size, static_buf) catch failure:
                failed = true
                recover ()
            if failed:
                break
    var atom = no_atom
    if not failed:
        atom = js_new_atom_len(s.ctx, buf, ident_pos) catch failure:
            failed = true
            recover no_atom
    if buf != static_buf:
        js_free(s.ctx, buf)
    *pp = p
    if failed:
        error(exception, "")
    return atom

# The characters of a JSON string into `b`, from the byte after the opening quote:
# answers the position after the closing quote.
func json_parse_string_chars(s: ParseState*, b: StringBuffer*, p0: const u8*, sep: i32) -> const u8*!:
    var p = p0
    while true:
        if p >= s.buf_end:
            try js_parse_error(s, "Unexpected end of JSON input")
        var c = (u32)*p
        p += 1
        if c == (u32)sep:
            break
        if c < 0x20:
            try js_parse_error_pos(s, p - 1, "Bad control character in string literal")
        if c == (u32)'\\':
            c = (u32)*p
            p += 1
            match (u8)c:
                'b':
                    c = 0x08
                'f':
                    c = 0x0c
                'n':
                    c = (u32)'\n'
                'r':
                    c = (u32)'\r'
                't':
                    c = (u32)'\t'
                '\\', '/':
                    # the character itself
                    c = c
                'u':
                    c = 0
                    for _i in 0..<4:
                        let h = cutils.from_hex((i32)*p)
                        p += 1
                        if h < 0:
                            try js_parse_error_pos(s, p - 1, "Bad Unicode escape")
                        c = (c << 4) | (u32)h
                '\n':
                    if s.ext_json:
                        continue
                    try js_parse_error_pos(s, p - 1, "Bad escaped character")
                'v':
                    if not s.ext_json:
                        try js_parse_error_pos(s, p - 1, "Bad escaped character")
                    c = 0x0b
                _:
                    if c != (u32)sep:
                        if p > s.buf_end:
                            try js_parse_error(s, "Unexpected end of JSON input")
                        # bad_escape:
                        try js_parse_error_pos(s, p - 1, "Bad escaped character")
        elif c >= 0x80:
            var p_next = p
            c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p_next)
            if c > 0x10FFFF:
                try js_parse_error_pos(s, p - 1, "Bad UTF-8 sequence")
            p = p_next
        try string_buffer_putc(b, c)
    return p

# A JSON string token delimited by `sep` from `*pp` (after the opening quote), which
# moves past it.
func json_parse_string(s: ParseState*, pp: const u8**, sep: i32) -> !:
    var b: StringBuffer = ---
    string_buffer_init(s.ctx, &b, 32) catch failure:
        string_buffer_free(&b)
        error(failure.code, failure.message)
    let p = json_parse_string_chars(s, &b, *pp, sep) catch failure:
        string_buffer_free(&b)
        error(failure.code, failure.message)
    s.token.val = TokenType.string
    s.token.u.str_.sep = sep
    # as QuickJS, a failure of string_buffer_end is not reported here
    s.token.u.str_.str = string_buffer_end(&b) catch failure:
        recover js_exception()
    *pp = p

# A SyntaxError naming the character at `p`.
func json_unexpected_token(s: ParseState*, p: const u8*) -> never!:
    try js_parse_error_pos(s, p, f"Unexpected token '{char((u32)*p)}'")

# A JSON number token at `*pp`, which moves past it.
func json_parse_number(s: ParseState*, pp: const u8**) -> !:
    var p = *pp
    let p_start = p
    var d: f64 = 0.0
    var done = false
    var atod_mem: dtoa.ATODTempMem = ---

    if *p == '+' or *p == '-':
        p += 1

    if not is_digit((i32)*p):
        if s.ext_json:
            if cutils.strstart(p, "Infinity", &p):
                d = f64.bits(0x7ff0000000000000)
                if *p_start == '-':
                    d = -d
                done = true
            elif cutils.strstart(p, "NaN", &p):
                d = f64.bits(0x7ff8000000000000)
                done = true
            elif *p != '.':
                try json_unexpected_token(s, p)
        else:
            try json_unexpected_token(s, p)

    if not done and p[0] == '0':
        if s.ext_json:
            # also accepts base 16, 8 and 2 prefix for integers
            var radix: i32 = 10
            if p[1] == 'x' or p[1] == 'X':
                p += 2
                radix = 16
            elif p[1] == 'o' or p[1] == 'O':
                p += 2
                radix = 8
            elif p[1] == 'b' or p[1] == 'B':
                p += 2
                radix = 2
            if radix != 10:
                # prefix is present
                if to_digit((i32)*p) >= radix:
                    try json_unexpected_token(s, p)
                d = dtoa.js_atod(p_start, &p, 0, dtoa.atod_int_only | dtoa.atod_accept_bin_oct,
                                 &atod_mem)
                done = true
        if not done and is_digit((i32)p[1]):
            try js_parse_error_pos(s, p, "Unexpected number")

    if not done:
        while is_digit((i32)*p):
            p += 1

        if *p == '.':
            p += 1
            if not is_digit((i32)*p):
                try js_parse_error_pos(s, p, "Unterminated fractional number")
            while is_digit((i32)*p):
                p += 1
        if *p == 'e' or *p == 'E':
            p += 1
            if *p == '+' or *p == '-':
                p += 1
            if not is_digit((i32)*p):
                try js_parse_error_pos(s, p, "Exponent part is missing a number")
            while is_digit((i32)*p):
                p += 1
        d = dtoa.js_atod(p_start, none, 10, 0, &atod_mem)
    # done:
    s.token.val = TokenType.number
    s.token.u.num.val = js_new_float64(d)
    *pp = p

# mark: json_next_token =========================================================================

# Read the next JSON token (or JSON superset token when s.ext_json) into s.token.
func json_next_token(s: ParseState*) -> !:
    if js_check_stack_overflow(s.ctx.rt, 0):
        try js_parse_error(s, "stack overflow")

    free_token(s, &s.token)
    errdefer set_token_error(s)

    var p = s.buf_ptr
    s.last_ptr = p
    # redo:
    while true:
        s.token.ptr = p
        let c = *p
        match c:
            0:
                if p >= s.buf_end:
                    s.token.val = TokenType.eof
                else:
                    s.token.val = (TokenType)(i32)c
                    p += 1
            '\'':
                if not s.ext_json:
                    # JSON does not accept single quoted strings
                    s.token.val = (TokenType)(i32)c
                    p += 1
                else:
                    p += 1
                    try json_parse_string(s, &p, (i32)c)
            '"':
                p += 1
                try json_parse_string(s, &p, (i32)c)
            '\r', '\n':
                # accept DOS and MAC newline sequences
                if c == '\r' and p[1] == '\n':
                    p += 1
                p += 1
                continue
            0x0c, 0x0b:
                if not s.ext_json:
                    # JSONWhitespace does not match <VT>, nor <FF>
                    s.token.val = (TokenType)(i32)c
                    p += 1
                else:
                    p += 1
                    continue
            ' ', '\t':
                p += 1
                continue
            '/':
                if not s.ext_json:
                    # JSON does not accept comments
                    s.token.val = (TokenType)(i32)c
                    p += 1
                elif p[1] == '*':
                    # comment
                    p += 2
                    while true:
                        if *p == 0 and p >= s.buf_end:
                            try js_parse_error(s, "unexpected end of comment")
                        if p[0] == '*' and p[1] == '/':
                            p += 2
                            break
                        if *p >= 0x80:
                            let c1 = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
                            if c1 == -1:
                                # skip invalid UTF-8
                                p += 1
                        else:
                            p += 1
                    continue
                elif p[1] == '/':
                    # line comment
                    p = skip_line_comment(s, p + 2)
                    continue
                else:
                    s.token.val = (TokenType)(i32)c
                    p += 1
            'a'..='z', 'A'..='Z', '_', '$':
                p += 1
                let atom = try json_parse_ident(s, &p, (i32)c)
                s.token.u.ident.atom = atom
                s.token.u.ident.has_escape = false
                s.token.u.ident.is_reserved = false
                s.token.val = TokenType.ident
            '+':
                if not s.ext_json:
                    s.token.val = (TokenType)(i32)c
                    p += 1
                else:
                    try json_parse_number(s, &p)
            '.':
                if s.ext_json and is_digit((i32)p[1]):
                    try json_parse_number(s, &p)
                else:
                    s.token.val = (TokenType)(i32)c
                    p += 1
            '-', '0'..='9':
                # number
                try json_parse_number(s, &p)
            _:
                if c >= 128:
                    try js_parse_error(s, "unexpected character")
                # def_token:
                s.token.val = (TokenType)(i32)c
                p += 1
        break
    s.buf_ptr = p

# dump_token(s, &s.token)

# mark: Look-ahead ==============================================================================

# Whether the identifier at `p` is `text` followed by a character that cannot continue
# an identifier.
func match_identifier(p0: const u8*, text: str) -> bool:
    var p = p0
    for ch in text.bytes:
        if ch != *p:
            return false
        p += 1
    var c = (u32)*p
    if c >= 128:
        c = (u32)cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
    return not unicode.lre_js_is_ident_next(c)

# simple_next_token() is used to check for the next token in simple cases.
# It is only used for ':' and '=>', 'let' or 'function' look-ahead.
# (*pp) is only set if TOK_IMPORT is returned for JS_DetectModule()
# Whitespace and comments are skipped correctly.
# Then the next token is analyzed, only for specific words.
# Return values:
# - '\n' if !no_line_terminator
# - TOK_ARROW, TOK_IN, TOK_IMPORT, TOK_OF, TOK_EXPORT, TOK_FUNCTION
# - TOK_IDENT is returned for other identifiers and keywords
# - otherwise the next character or unicode codepoint is returned.
func simple_next_token(pp: const u8**, no_line_terminator: bool) -> TokenType:
    # skip spaces and comments
    var p = *pp
    while true:
        let b = *p
        p += 1
        match b:
            '\r', '\n':
                if no_line_terminator:
                    return TokenType.newline
                continue
            ' ', '\t', 0x0b, 0x0c:
                continue
            '/':
                if *p == '/':
                    if no_line_terminator:
                        return TokenType.newline
                    while *p != 0 and *p != '\r' and *p != '\n':
                        p += 1
                    continue
                if *p == '*':
                    while true:
                        p += 1
                        if *p == 0:
                            break
                        if (*p == '\r' or *p == '\n') and no_line_terminator:
                            return TokenType.newline
                        if *p == '*' and p[1] == '/':
                            p += 2
                            break
                    continue
                return (TokenType)(i32)b
            '=':
                if *p == '>':
                    return TokenType.arrow
                return (TokenType)(i32)b
            'i':
                if match_identifier(p, "n"):
                    return TokenType.in_
                if match_identifier(p, "mport"):
                    *pp = p + 5
                    return TokenType.import_
                return TokenType.ident
            'o':
                if match_identifier(p, "f"):
                    return TokenType.of
                return TokenType.ident
            'e':
                if match_identifier(p, "xport"):
                    return TokenType.export_
                return TokenType.ident
            'f':
                if match_identifier(p, "unction"):
                    return TokenType.function
                return TokenType.ident
            '\\':
                if *p == 'u':
                    if unicode.lre_js_is_ident_first((u32)lre_parse_escape(&p, 1)):
                        return TokenType.ident
                return (TokenType)(i32)b
            _:
                var c = (u32)b
                if c >= 128:
                    c = (u32)cutils.unicode_from_utf8(p - 1, cutils.utf8_char_len_max, &p)
                    if no_line_terminator and (c == cp_ps or c == cp_ls):
                        return TokenType.newline
                if unicode.lre_is_space(c):
                    continue
                if unicode.lre_js_is_ident_first(c):
                    return TokenType.ident
                return (TokenType)(i32)c

# The kind of the token after the current one, by simple_next_token.
func peek_token(s: ParseState*, no_line_terminator: bool) -> TokenType:
    var p = s.buf_ptr
    return simple_next_token(&p, no_line_terminator)

# Move `*pp` past a "#!" line at the start of the source.
func skip_shebang(pp: const u8**, buf_end: const u8*):
    var p = *pp
    if p[0] == '#' and p[1] == '!':
        p += 2
        while p < buf_end:
            if *p == '\n' or *p == '\r':
                break
            elif *p >= 0x80:
                let c = cutils.unicode_from_utf8(p, cutils.utf8_char_len_max, &p)
                if c == (i32)cp_ls or c == (i32)cp_ps:
                    break
                elif c == -1:
                    # skip invalid UTF-8
                    p += 1
            else:
                p += 1
        *pp = p

## Whether `input` (`input_len` bytes, NUL-terminated) is the source of a module, by a
## heuristic: after comments, the 'import' keyword not followed by '(' or '.', or the
## 'export' keyword (JS_DetectModule).
pub func js_detect_module(input: const u8*, input_len: usize) -> bool:
    var p = input
    skip_shebang(&p, p + input_len)
    match simple_next_token(&p, false):
        .import_:
            let tok = simple_next_token(&p, false)
            return tok != TokenType.dot and tok != TokenType.lparen
        .export_:
            return true
        _:
            return false